From fdefd4be9398b9610a539360353fd61b521732d4 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 27 May 2025 15:52:57 +0200 Subject: [PATCH 01/33] feat: duckdb sql lang support (#5761) --- backend/.gitignore | 3 +- backend/Cargo.lock | 376 +++++++--- backend/Cargo.toml | 4 +- backend/ee-repo-ref.txt | 2 +- .../20250515084520_duckdb_support.down.sql | 1 + .../20250515084520_duckdb_support.up.sql | 3 + .../parsers/windmill-parser-sql/src/lib.rs | 74 ++ .../parsers/windmill-parser-wasm/src/lib.rs | 6 + backend/windmill-api/openapi.yaml | 3 +- backend/windmill-api/src/workspaces_export.rs | 1 + backend/windmill-common/src/client.rs | 46 +- backend/windmill-common/src/s3_helpers.rs | 17 + backend/windmill-common/src/schema.rs | 2 +- backend/windmill-common/src/scripts.rs | 5 +- backend/windmill-common/src/worker.rs | 1 + backend/windmill-worker/Cargo.toml | 2 + .../windmill-worker/src/duckdb_executor.rs | 664 ++++++++++++++++++ backend/windmill-worker/src/lib.rs | 2 + backend/windmill-worker/src/mysql_executor.rs | 14 +- backend/windmill-worker/src/pg_executor.rs | 16 +- .../windmill-worker/src/windmill-client.js | 2 + backend/windmill-worker/src/worker.rs | 28 + cli/bootstrap/script_bootstrap.ts | 3 +- cli/build.sh | 1 + cli/metadata.ts | 3 + cli/script.ts | 10 +- cli/script_common.ts | 7 +- cli/sync.ts | 1 + cli/wasm/regex/windmill_parser_wasm.js | 1 + frontend/package-lock.json | 8 +- frontend/package.json | 2 +- frontend/src/lib/components/Editor.svelte | 5 +- .../src/lib/components/HighlightCode.svelte | 8 +- .../common/languageIcons/LanguageIcon.svelte | 9 +- .../lib/components/icons/DuckDbIcon.svelte | 19 + frontend/src/lib/consts.ts | 50 ++ frontend/src/lib/infer.ts | 10 +- frontend/src/lib/inferArgSig.ts | 20 +- frontend/src/lib/script_helpers.ts | 18 + frontend/src/lib/scripts.ts | 7 +- .../(root)/(logged)/run/[...run]/+page.svelte | 2 +- 41 files changed, 1310 insertions(+), 146 deletions(-) create mode 100644 backend/migrations/20250515084520_duckdb_support.down.sql create mode 100644 backend/migrations/20250515084520_duckdb_support.up.sql create mode 100644 backend/windmill-worker/src/duckdb_executor.rs create mode 100644 frontend/src/lib/components/icons/DuckDbIcon.svelte diff --git a/backend/.gitignore b/backend/.gitignore index 4ca669fc53..f75b9d1eaa 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -5,4 +5,5 @@ oauth2.json tracing.folded heaptrack* index/ -windmill-api/openapi-*.* \ No newline at end of file +windmill-api/openapi-*.* +.duckdb/* \ No newline at end of file diff --git a/backend/Cargo.lock b/backend/Cargo.lock index eaf6ef8e37..8f6ca407af 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -286,25 +286,57 @@ dependencies = [ "serde", ] +[[package]] +name = "arrow" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" +dependencies = [ + "arrow-arith 54.3.1", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-cast 54.3.1", + "arrow-data 54.3.1", + "arrow-ord 54.3.1", + "arrow-row 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "arrow-string 54.3.1", +] + [[package]] name = "arrow" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-arith 55.1.0", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", "arrow-csv", - "arrow-data", + "arrow-data 55.1.0", "arrow-ipc", "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-ord 55.1.0", + "arrow-row 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", + "arrow-string 55.1.0", +] + +[[package]] +name = "arrow-arith" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "num", ] [[package]] @@ -313,14 +345,30 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "num", ] +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "chrono", + "half", + "hashbrown 0.15.3", + "num", +] + [[package]] name = "arrow-array" version = "55.1.0" @@ -328,9 +376,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f" dependencies = [ "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "chrono-tz", "half", @@ -338,6 +386,17 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-buffer" version = "55.1.0" @@ -349,17 +408,38 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-cast" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + [[package]] name = "arrow-cast" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "atoi", "base64 0.22.1", "chrono", @@ -376,9 +456,9 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-cast 55.1.0", + "arrow-schema 55.1.0", "chrono", "csv", "csv-core", @@ -386,14 +466,26 @@ dependencies = [ "regex", ] +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer 54.3.1", + "arrow-schema 54.3.1", + "half", + "num", +] + [[package]] name = "arrow-data" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 55.1.0", + "arrow-schema 55.1.0", "half", "num", ] @@ -404,10 +496,10 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "flatbuffers", "lz4_flex", ] @@ -418,11 +510,11 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "chrono", "half", "indexmap 2.9.0", @@ -434,17 +526,43 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-ord" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", +] + [[package]] name = "arrow-ord" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", +] + +[[package]] +name = "arrow-row" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "half", ] [[package]] @@ -453,19 +571,42 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "half", ] +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" +dependencies = [ + "bitflags 2.9.1", +] + [[package]] name = "arrow-schema" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" +[[package]] +name = "arrow-select" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" +dependencies = [ + "ahash 0.8.12", + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "num", +] + [[package]] name = "arrow-select" version = "55.1.0" @@ -473,24 +614,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", "num", ] +[[package]] +name = "arrow-string" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" +dependencies = [ + "arrow-array 54.3.1", + "arrow-buffer 54.3.1", + "arrow-data 54.3.1", + "arrow-schema 54.3.1", + "arrow-select 54.3.1", + "memchr", + "num", + "regex", + "regex-syntax 0.8.5", +] + [[package]] name = "arrow-string" version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-data 55.1.0", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "memchr", "num", "regex", @@ -1838,6 +1996,12 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbc" version = "0.1.2" @@ -2599,9 +2763,9 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "arrow", + "arrow 55.1.0", "arrow-ipc", - "arrow-schema", + "arrow-schema 55.1.0", "async-trait", "bytes", "bzip2", @@ -2654,7 +2818,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -2680,7 +2844,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -2704,7 +2868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "arrow-ipc", "base64 0.22.1", "half", @@ -2738,7 +2902,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ - "arrow", + "arrow 55.1.0", "async-compression", "async-trait", "bytes", @@ -2774,7 +2938,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2799,7 +2963,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2824,7 +2988,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "bytes", "datafusion-catalog", @@ -2861,7 +3025,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ - "arrow", + "arrow 55.1.0", "dashmap 6.1.0", "datafusion-common", "datafusion-expr", @@ -2880,7 +3044,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ - "arrow", + "arrow 55.1.0", "chrono", "datafusion-common", "datafusion-doc", @@ -2901,7 +3065,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ - "arrow", + "arrow 55.1.0", "datafusion-common", "indexmap 2.9.0", "itertools 0.14.0", @@ -2914,8 +3078,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 55.1.0", + "arrow-buffer 55.1.0", "base64 0.22.1", "blake2", "blake3", @@ -2944,7 +3108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2965,7 +3129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", @@ -2977,8 +3141,8 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ - "arrow", - "arrow-ord", + "arrow 55.1.0", + "arrow-ord 55.1.0", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -2998,7 +3162,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "datafusion-catalog", "datafusion-common", @@ -3052,7 +3216,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ - "arrow", + "arrow 55.1.0", "chrono", "datafusion-common", "datafusion-expr", @@ -3072,7 +3236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -3094,7 +3258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash 0.8.12", - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -3107,7 +3271,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ - "arrow", + "arrow 55.1.0", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -3127,9 +3291,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 55.1.0", + "arrow-ord 55.1.0", + "arrow-schema 55.1.0", "async-trait", "chrono", "datafusion-common", @@ -3156,7 +3320,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" dependencies = [ - "arrow", + "arrow 55.1.0", "async-trait", "dashmap 6.1.0", "datafusion-common", @@ -3180,7 +3344,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ - "arrow", + "arrow 55.1.0", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -4640,6 +4804,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "duckdb" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ac283b6621e3becf8014d1efa655522794075834c72f744573debef9c9f6c8" +dependencies = [ + "arrow 54.3.1", + "cast", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink 0.9.1", + "libduckdb-sys", + "memchr", + "num-integer", + "rust_decimal", + "smallvec", + "strum 0.25.0", +] + [[package]] name = "dunce" version = "1.0.5" @@ -7149,6 +7332,22 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "libduckdb-sys" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cac9d03484c43fefac8b2066a253c9b0b3b0cd02cbe02a9ea2312f7e382618" +dependencies = [ + "autocfg", + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "vcpkg", +] + [[package]] name = "libffi" version = "3.2.0" @@ -8950,13 +9149,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231" dependencies = [ "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", + "arrow-array 55.1.0", + "arrow-buffer 55.1.0", + "arrow-cast 55.1.0", + "arrow-data 55.1.0", "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-schema 55.1.0", + "arrow-select 55.1.0", "base64 0.22.1", "brotli 8.0.1", "bytes", @@ -15056,6 +15255,7 @@ dependencies = [ "deno_web", "deno_webidl", "dotenv", + "duckdb", "dyn-iter", "flume", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 069aca4367..0273aa2c59 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -87,13 +87,14 @@ python = ["windmill-worker/python", "windmill-api/python"] rust = ["windmill-worker/rust"] mysql = ["windmill-worker/mysql"] oracledb = ["windmill-worker/oracledb"] +duckdb = ["windmill-worker/duckdb"] mssql = ["windmill-worker/mssql"] bigquery = ["windmill-worker/bigquery"] php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] -all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"] +all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"] [patch.crates-io] @@ -238,6 +239,7 @@ json-pointer = "^0" itertools = "^0" regex = "^1" semver = "^1" +duckdb = { version = "1.2.2", features = ["bundled"] } v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix deno_fetch = "0.214.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 04b305d5a2..be1e897bcd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6899b8151329218a1df59964dac57e0e004ae25a \ No newline at end of file +32039f675060b5996951708368bdefe14278d5cd \ No newline at end of file diff --git a/backend/migrations/20250515084520_duckdb_support.down.sql b/backend/migrations/20250515084520_duckdb_support.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250515084520_duckdb_support.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250515084520_duckdb_support.up.sql b/backend/migrations/20250515084520_duckdb_support.up.sql new file mode 100644 index 0000000000..869fcc07dc --- /dev/null +++ b/backend/migrations/20250515084520_duckdb_support.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'duckdb'; +UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["duckdb"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java"]}'::jsonb AND NOT config->'worker_tags' @> '"duckdb"'::jsonb; diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 21ebe45f7b..0fe72f0296 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -83,6 +83,21 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result { } } +pub fn parse_duckdb_sig(code: &str) -> anyhow::Result { + let parsed = parse_duckdb_file(&code)?; + if let Some(args) = parsed { + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) + } else { + Err(anyhow!("Error parsing sql".to_string())) + } +} + pub fn parse_snowflake_sig(code: &str) -> anyhow::Result { let parsed = parse_snowflake_file(&code)?; if let Some(x) = parsed { @@ -212,6 +227,9 @@ lazy_static::lazy_static! { // -- @name (type) = default static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + // -- $name (type) = default + static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -577,6 +595,35 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +fn parse_duckdb_file(code: &str) -> anyhow::Result>> { + let mut args: Vec = vec![]; + + for cap in RE_ARG_DUCKDB.captures_iter(code) { + let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap(); + let typ = cap + .get(2) + .map(|x| x.as_str().to_string().to_lowercase()) + .unwrap(); + let default = cap.get(3).map(|x| x.as_str().to_string()); + let has_default = default.is_some(); + let parsed_typ = parse_duckdb_typ(typ.as_str()); + + let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x)); + + args.push(Arg { + name, + typ: parsed_typ, + default: parsed_default, + otyp: Some(typ), + has_default, + oidx: None, + }); + } + + args.append(&mut parse_sql_sanitized_interpolation(code)); + Ok(Some(args)) +} + fn parse_snowflake_file(code: &str) -> anyhow::Result>> { let mut args: Vec = vec![]; @@ -729,6 +776,33 @@ pub fn parse_bigquery_typ(typ: &str) -> Typ { } } +pub fn parse_duckdb_typ(typ: &str) -> Typ { + if typ.ends_with("[]") { + let base_typ = parse_duckdb_typ(typ.strip_suffix("[]").unwrap()); + Typ::List(Box::new(base_typ)) + } else { + match typ { + "varchar" | "char" | "bpchar" | "text" | "string" => Typ::Str(None), + "blob" | "bytea" | "binary" | "varbinary" | "bitstring" => Typ::Bytes, + "boolean" | "bool" | "bit" | "logical" => Typ::Bool, + "bigint" | "int8" | "long" | "integer" | "int4" | "int" | "smallint" | "int2" + | "short" | "tinyint" | "int1" | "signed" | "ubigint" | "uhugeint" | "uinteger" + | "usmallint" | "utinyint" => Typ::Int, + "decimal" | "numeric" | "double" | "float8" | "float" | "float4" | "real" => Typ::Float, + "date" + | "time" + | "timestamp with time zone" + | "timestamptz" + | "timestamp" + | "datetime" => Typ::Datetime, + "uuid" | "json" => Typ::Str(None), + "interval" | "hugeint" => Typ::Str(None), + "s3object" => Typ::Resource("S3Object".to_string()), + _ => Typ::Str(None), + } + } +} + pub fn parse_snowflake_typ(typ: &str) -> Typ { match typ { "varchar" => Typ::Str(None), diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index cd280e7af5..4270c28c60 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -96,6 +96,12 @@ pub fn parse_oracledb(code: &str) -> String { wrap_sig(windmill_parser_sql::parse_oracledb_sig(code)) } +#[cfg(feature = "sql-parser")] +#[wasm_bindgen] +pub fn parse_duckdb(code: &str) -> String { + wrap_sig(windmill_parser_sql::parse_duckdb_sig(code)) +} + #[cfg(feature = "sql-parser")] #[wasm_bindgen] pub fn parse_bigquery(code: &str) -> String { diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fd3e81f209..20f495568d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14242,7 +14242,8 @@ components: ansible, csharp, nu, - java + java, + duckdb # for related places search: ADD_NEW_LANG ] diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index c58d773f66..cd2be0f708 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -360,6 +360,7 @@ pub(crate) async fn tarball_workspace( ScriptLang::Bigquery => "bq.sql", ScriptLang::Snowflake => "sf.sql", ScriptLang::Mssql => "ms.sql", + ScriptLang::DuckDb => "duckdb.sql", ScriptLang::Graphql => "gql", ScriptLang::Nativets => "fetch.ts", ScriptLang::Bun | ScriptLang::Bunnative => { diff --git a/backend/windmill-common/src/client.rs b/backend/windmill-common/src/client.rs index 95d9f64bd2..0c5394c6e6 100644 --- a/backend/windmill-common/src/client.rs +++ b/backend/windmill-common/src/client.rs @@ -2,7 +2,11 @@ use anyhow::Context; use reqwest::{Body, Response}; use serde::de::DeserializeOwned; -use crate::utils::HTTP_CLIENT; +use crate::{ + error::{self, to_anyhow}, + s3_helpers::{DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse}, + utils::HTTP_CLIENT, +}; #[derive(Clone)] pub struct AuthedClient { @@ -197,4 +201,44 @@ impl AuthedClient { _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, } } + + pub async fn get_duckdb_connection_settings( + &self, + s3: &DuckdbConnectionSettingsQueryV2, + ) -> error::Result { + let url = format!( + "{}/api/w/{}/job_helpers/v2/duckdb_connection_settings", + self.base_internal_url, &self.workspace + ); + let response = self + .force_client + .as_ref() + .unwrap_or(&HTTP_CLIENT) + .post(url) + .header( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::ACCEPT, + reqwest::header::HeaderValue::from_static("application/json"), + ) + .header( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token)) + .map_err(|e| error::Error::BadConfig(e.to_string()))?, + ) + .body(serde_json::to_string(&s3).map_err(to_anyhow)?) + .send() + .await + .context(format!("Sent get_duckdb_connection_settings request",)) + .map_err(error::Error::from)?; + match response.status().as_u16() { + 200u16 => Ok(response + .json::() + .await + .context("decoding duckdb_connection_settings response as json")?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?, + } + } } diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 29f59cf07c..7e7c48851e 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -891,3 +891,20 @@ pub async fn convert_json_line_stream>( Ok(tokio_stream::wrappers::ReceiverStream::new(rx)) } + +#[derive(Deserialize, Serialize)] +pub struct DuckdbConnectionSettingsResponse { + pub connection_settings_str: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub azure_container_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_bucket: Option, +} + +#[derive(Deserialize, Serialize)] +pub struct DuckdbConnectionSettingsQueryV2 { + #[serde(skip_serializing_if = "Option::is_none")] + pub s3_resource_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub storage: Option, +} diff --git a/backend/windmill-common/src/schema.rs b/backend/windmill-common/src/schema.rs index 2b24ead0d3..71ecd220d1 100644 --- a/backend/windmill-common/src/schema.rs +++ b/backend/windmill-common/src/schema.rs @@ -349,7 +349,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool { let comment = match lang { Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//", Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#", - Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--", + Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--", Rust => "//!", // for related places search: ADD_NEW_LANG }; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index e0391767f8..dda9daf0f4 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -46,13 +46,13 @@ pub enum ScriptLang { Graphql, Mssql, OracleDB, + DuckDb, Php, Rust, Ansible, CSharp, Nu, - Java, - // for related places search: ADD_NEW_LANG + Java, // for related places search: ADD_NEW_LANG } impl ScriptLang { @@ -73,6 +73,7 @@ impl ScriptLang { ScriptLang::Mssql => "mssql", ScriptLang::Graphql => "graphql", ScriptLang::OracleDB => "oracledb", + ScriptLang::DuckDb => "duckdb", ScriptLang::Php => "php", ScriptLang::Rust => "rust", ScriptLang::Ansible => "ansible", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index aa1a715758..6b11fa086f 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -79,6 +79,7 @@ lazy_static::lazy_static! { "csharp".to_string(), "nu".to_string(), "java".to_string(), + "duckdb".to_string(), // for related places search: ADD_NEW_LANG "dependency".to_string(), "flow".to_string(), diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 4dccc3213c..dbc8f2ab27 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -31,6 +31,7 @@ csharp = ["dep:windmill-parser-csharp"] rust = ["dep:windmill-parser-rust"] nu = ["dep:windmill-parser-nu"] java = ["dep:windmill-parser-java"] +duckdb = [] [dependencies] windmill-queue.workspace = true @@ -92,6 +93,7 @@ deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } async-stream.workspace = true +duckdb.workspace = true postgres-native-tls.workspace = true native-tls.workspace = true diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs new file mode 100644 index 0000000000..8ee78292d3 --- /dev/null +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -0,0 +1,664 @@ +use std::collections::HashMap; +use std::env; + +use duckdb::types::TimeUnit; +use duckdb::{params_from_iter, Row}; +use rust_decimal::prelude::FromPrimitive; +use rust_decimal::Decimal; +use serde_json::value::RawValue; +use serde_json::{json, Value}; +use tokio::fs::remove_file; +use tokio::task; +use uuid::Uuid; +use windmill_common::error::{to_anyhow, Error, Result}; +use windmill_common::s3_helpers::{ + DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object, +}; +use windmill_common::worker::{to_raw_value, Connection}; +use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; +use windmill_queue::{CanceledBy, MiniPulledJob}; + +use crate::common::{build_args_values, OccupancyMetrics}; +use crate::handle_child::run_future_with_polling_update_job_poller; +#[cfg(feature = "mysql")] +use crate::mysql_executor::MysqlDatabase; +use crate::pg_executor::PgDatabase; +use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args; +use windmill_common::client::AuthedClient; + +fn do_duckdb_inner( + conn: &duckdb::Connection, + query: &str, + job_args: &HashMap, + skip_collect: bool, + column_order: &mut Option>, +) -> Result> { + let mut rows_vec = vec![]; + + let (query, job_args) = interpolate_named_args(query, &job_args); + + let mut stmt = conn + .prepare(&query) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + let mut rows = stmt + .query(params_from_iter(job_args)) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + if skip_collect { + return Ok(to_raw_value(&json!([]))); + } + + // Statement needs to be stepped at least once or stmt.column_names() will panic + let mut column_names = None; + loop { + let row = rows.next(); + match row { + Ok(Some(row)) => { + // Set column names if not already set + let stmt = row.as_ref(); + let column_names = match column_names.as_ref() { + Some(column_names) => column_names, + None => { + column_names = Some(stmt.column_names()); + column_names.as_ref().unwrap() + } + }; + + let row = row_to_value(row, &column_names.as_slice()) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + rows_vec.push(row); + } + Ok(None) => break, + Err(e) => { + return Err(Error::ExecutionErr(e.to_string())); + } + } + } + + if let (Some(column_order), Some(column_names)) = (column_order.as_mut(), column_names) { + *column_order = column_names.clone(); + } + + return Ok(to_raw_value(&rows_vec)); +} + +pub async fn do_duckdb( + job: &MiniPulledJob, + client: &AuthedClient, + query: &str, + conn: &Connection, + mem_peak: &mut i32, + canceled_by: &mut Option, + worker_name: &str, + column_order_ref: &mut Option>, + occupancy_metrics: &mut OccupancyMetrics, +) -> Result> { + let result_f = async { + let sig = parse_duckdb_sig(query)?.args; + let mut job_args = build_args_values(job, client, conn).await?; + + let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?; + // Prevent interpolate_named_args from detecting argument identifiers in the signature for + // the first query block + let query = trunc_sig(query); + + let (_query_with_transformed_s3_uris, mut used_storages) = + transform_s3_uris(query, client).await?; + let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query); + + let job_args = { + let mut m: HashMap = HashMap::new(); + for sig_arg in sig.into_iter() { + let json_value = job_args + .remove(&sig_arg.name) + .or_else(|| sig_arg.default) + .unwrap_or_else(|| json!(null)); + + if matches!(&sig_arg.otyp.as_ref().map(String::as_str), Some("s3object")) { + let s3_obj = serde_json::from_value::(json_value).map_err(|e| { + Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e)) + })?; + let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client + .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 { + s3_resource_path: None, + storage: s3_obj.storage.clone(), + }) + .await?; + + let uri = match ( + &duckdb_conn_settings.s3_bucket, + &duckdb_conn_settings.azure_container_path, + ) { + (Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3), + (None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3), + _ => { + return Err(Error::ExecutionErr( + "S3Object must have either s3_bucket or azure_container_path" + .to_string(), + )); + } + }; + m.insert(sig_arg.name, duckdb::types::Value::Text(uri)); + used_storages.insert(s3_obj.storage, duckdb_conn_settings); + } else { + let duckdb_value = json_value_to_duckdb_value( + &json_value, + sig_arg + .otyp + .clone() + .unwrap_or_else(|| "text".to_string()) + .as_str(), + client, + )?; + m.insert(sig_arg.name, duckdb_value); + } + } + m + }; + + let query_block_list = parse_sql_blocks(query); + + // Replace windmill resource ATTACH statements with the real instructions + let query_block_list = { + let mut v = vec![]; + for query_block in query_block_list.iter() { + match parse_attach_db_resource(query_block) { + Some(parsed) => v.extend( + transform_attach_db_resource_query(&parsed, &job.id, client).await?, + ), + None => v.push(query_block.to_string()), + }; + } + v + }; + + // duckdb::Connection is not Send so we do it in a single blocking task + let (result, column_order) = task::spawn_blocking(move || { + let conn = duckdb::Connection::open_in_memory() + .map_err(|e| Error::ConnectingToDatabase(e.to_string()))?; + + for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in + used_storages.into_iter() + { + conn.execute_batch(&connection_settings_str) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + } + + let mut result: Option> = None; + let mut column_order = None; + for (query_block_index, query_block) in query_block_list.iter().enumerate() { + result = Some( + do_duckdb_inner( + &conn, + query_block.as_str(), + &job_args, + query_block_index != query_block_list.len() - 1, + &mut column_order, + ) + .map_err(|e| Error::ExecutionErr(e.to_string()))?, + ); + } + let result = result.unwrap_or_else(|| to_raw_value(&json!([]))); + Ok::<_, Error>((result, column_order)) + }) + .await + .map_err(to_anyhow)??; + + *column_order_ref = column_order; + + // BigQuery cleanup + let bq_credentials_path = make_bq_credentials_path(&job.id); + env::remove_var("GOOGLE_APPLICATION_CREDENTIALS"); + if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) { + remove_file(&bq_credentials_path).await.map_err(to_anyhow)?; + } + Ok(result) + }; + + let result = run_future_with_polling_update_job_poller( + job.id, + job.timeout, + conn, + mem_peak, + canceled_by, + result_f, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), + ) + .await?; + + Ok(result) +} + +fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result> { + let mut obj = serde_json::Map::new(); + for (i, key) in column_names.iter().enumerate() { + let value: duckdb::types::Value = + row.get(i).map_err(|e| Error::ExecutionErr(e.to_string()))?; + let json_value = match value { + duckdb::types::Value::Null => serde_json::Value::Null, + duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b), + duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()), + duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()), + duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), + duckdb::types::Value::Float(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f as f64) + .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?, + ), + duckdb::types::Value::Double(f) => serde_json::Value::Number( + serde_json::Number::from_f64(f) + .ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?, + ), + duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), + duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Text(s) => serde_json::Value::String(s), + duckdb::types::Value::Blob(b) => serde_json::Value::Array( + b.into_iter() + .map(|byte| serde_json::Value::Number(byte.into())) + .collect(), + ), + duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), + duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ + "months": months, + "days": days, + "nanos": nanos + }), + duckdb::types::Value::List(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| serde_json::Value::String(format!("{:?}", v))) + .collect(), + ), + duckdb::types::Value::Enum(e) => serde_json::Value::String(e), + duckdb::types::Value::Struct(fields) => serde_json::Value::Object( + fields + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v)))) + .collect(), + ), + duckdb::types::Value::Array(values) => serde_json::Value::Array( + values + .into_iter() + .map(|v| serde_json::Value::String(format!("{:?}", v))) + .collect(), + ), + duckdb::types::Value::Map(map) => serde_json::Value::Object( + map.iter() + .map(|(k, v)| { + ( + format!("{:?}", k), + serde_json::Value::String(format!("{:?}", v)), + ) + }) + .collect(), + ), + duckdb::types::Value::Union(value) => { + serde_json::Value::String(format!("{:?}", *value)) + } + }; + obj.insert(key.clone(), json_value); + } + serde_json::value::to_raw_value(&obj).map_err(|e| e.into()) +} + +fn json_value_to_duckdb_value( + json_value: &serde_json::Value, + arg_type: &str, + client: &AuthedClient, +) -> Result { + let arg_type = arg_type.to_lowercase(); + let duckdb_value = match json_value { + serde_json::Value::Null => duckdb::types::Value::Null, + serde_json::Value::Bool(b) => duckdb::types::Value::Boolean(*b), + + serde_json::Value::String(s) + if matches!( + arg_type.as_str(), + "timestamp" | "timestamptz" | "timestamp with time zone" | "datetime" + ) => + { + string_to_duckdb_timestamp(&s)? + } + serde_json::Value::String(s) if arg_type.as_str() == "date" => string_to_duckdb_date(&s)?, + serde_json::Value::String(s) if arg_type.as_str() == "time" => string_to_duckdb_time(&s)?, + serde_json::Value::String(s) => duckdb::types::Value::Text(s.clone()), + + serde_json::Value::Number(n) if n.is_i64() => { + let v = n.as_i64().unwrap(); + match arg_type.as_str() { + "tinyint" | "int1" => duckdb::types::Value::TinyInt(v as i8), + "smallint" | "int2" | "short" => duckdb::types::Value::SmallInt(v as i16), + "integer" | "int4" | "int" | "signed" => duckdb::types::Value::Int(v as i32), + "bigint" | "int8" | "long" => duckdb::types::Value::BigInt(v), + "hugeint" => duckdb::types::Value::HugeInt(v as i128), + "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32), + "double" | "float8" => duckdb::types::Value::Double(v as f64), + _ => duckdb::types::Value::BigInt(v), // default fallback + } + } + + serde_json::Value::Number(n) if n.is_u64() => { + let v = n.as_u64().unwrap(); + match arg_type.as_str() { + "utinyint" => duckdb::types::Value::UTinyInt(v as u8), + "usmallint" => duckdb::types::Value::USmallInt(v as u16), + "uinteger" => duckdb::types::Value::UInt(v as u32), + "ubigint" | "uhugeint" => duckdb::types::Value::UBigInt(v), + _ => duckdb::types::Value::UBigInt(v), // default fallback + } + } + + serde_json::Value::Number(n) if n.is_f64() => { + let v = n.as_f64().unwrap(); + match arg_type.as_str() { + "float" | "float4" | "real" => duckdb::types::Value::Float(v as f32), + "double" | "float8" => duckdb::types::Value::Double(v), + "decimal" | "numeric" => { + duckdb::types::Value::Decimal(Decimal::from_f64(v).ok_or_else(|| { + Error::ExecutionErr("Could not convert f64 to Decimal".to_string()) + })?) + } + _ => duckdb::types::Value::Double(v), // default fallback + } + } + + serde_json::Value::Array(arr) => duckdb::types::Value::Array( + arr.iter() + .map(|val| json_value_to_duckdb_value(val, arg_type.as_str(), client)) + .collect::>>()?, + ), + serde_json::Value::Object(map) => duckdb::types::Value::Struct( + map.iter() + .map(|(k, v)| { + Ok::<_, Error>(( + k.clone(), + json_value_to_duckdb_value(v, arg_type.as_str(), client)?, + )) + }) + .collect::>>()? + .into(), + ), + + value @ _ => { + return Err(Error::ExecutionErr(format!( + "Unsupported type in query: {:?} and signature {arg_type:?}", + value + ))) + } + }; + Ok(duckdb_value) +} + +fn string_to_duckdb_timestamp(s: &str) -> Result { + let ts = chrono::DateTime::parse_from_rfc3339(s) + .map_err(|e: chrono::ParseError| Error::ExecutionErr(e.to_string()))?; + Ok(duckdb::types::Value::Timestamp( + TimeUnit::Millisecond, + ts.timestamp_millis(), + )) +} + +fn string_to_duckdb_date(s: &str) -> Result { + use chrono::Datelike; + let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap(); + Ok(duckdb::types::Value::Date32(date.num_days_from_ce())) +} + +fn string_to_duckdb_time(s: &str) -> Result { + use chrono::Timelike; + let time = chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").unwrap(); + Ok(duckdb::types::Value::Time64( + TimeUnit::Microsecond, + time.num_seconds_from_midnight() as i64, + )) +} + +struct ParsedAttachDbResource<'a> { + resource_path: &'a str, + name: &'a str, + db_type: &'a str, + extra_args: Option<&'a str>, +} +fn parse_attach_db_resource<'a>(query: &'a str) -> Option> { + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap(); + } + + for cap in RE.captures_iter(query) { + if let (Some(resource_path), Some(name), Some(db_type)) = + (cap.get(1), cap.get(2), cap.get(3)) + { + let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim()); + return Some(ParsedAttachDbResource { + resource_path: query[resource_path.start()..resource_path.end()].trim(), + name: query[name.start()..name.end()].trim(), + db_type: query[db_type.start()..db_type.end()].trim(), + extra_args, + }); + } + } + None +} + +async fn transform_attach_db_resource_query( + parsed: &ParsedAttachDbResource<'_>, + job_id: &Uuid, + client: &AuthedClient, +) -> Result> { + match parsed.db_type.to_lowercase().as_str() { + "postgres" => { + let resource: PgDatabase = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + + let attach_str = format!( + "ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});", + resource.dbname, + resource + .user + .map(|u| format!("user={}", u)) + .unwrap_or_default(), + resource.host, + resource + .password + .map(|p| format!("password={}", p)) + .unwrap_or_default(), + resource + .port + .map(|p| format!("port={}", p)) + .unwrap_or_default(), + parsed.name, + parsed.extra_args.unwrap_or("") + ); + + Ok(vec![ + "INSTALL postgres;".to_string(), + "LOAD postgres;".to_string(), + attach_str, + ]) + } + "mysql" => { + #[cfg(not(feature = "mysql"))] + return Err(Error::ExecutionErr( + "MySQL feature is not enabled".to_string(), + )); + + #[cfg(feature = "mysql")] + { + let resource: MysqlDatabase = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + + let attach_str = format!( + "ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});", + resource.database, + resource.host, + resource + .ssl + .map(|ssl| if ssl { "required" } else { "disabled" }) + .unwrap_or("preferred"), + resource + .password + .map(|p| format!("password={}", p)) + .unwrap_or_default(), + resource + .port + .map(|p| format!("port={}", p)) + .unwrap_or_default(), + resource + .user + .map(|u| format!("user={}", u)) + .unwrap_or_default(), + parsed.name, + parsed.extra_args.unwrap_or("") + ); + + Ok(vec![ + "INSTALL mysql;".to_string(), + "LOAD mysql;".to_string(), + attach_str, + ]) + } + } + "bigquery" => { + let resource: Value = client + .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) + .await?; + // duckdb's bigquery extension requires a json file as credentials + let bq_credentials_path = make_bq_credentials_path(job_id); + env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path); + tokio::fs::write(&bq_credentials_path, resource.to_string()) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to write BigQuery credentials to {}: {}", + &bq_credentials_path, e + )) + })?; + let project_id: String = serde_json::from_value( + resource + .get("project_id") + .ok_or_else(|| { + Error::ExecutionErr("BigQuery resource must contain project_id".to_string()) + })? + .to_owned(), + ) + .map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?; + let attach_str = format!( + "ATTACH 'project={}' as {} (TYPE bigquery{});", + project_id, + parsed.name, + parsed.extra_args.unwrap_or("") + ) + .to_string(); + Ok(vec![ + "INSTALL bigquery FROM community;".to_string(), + "LOAD bigquery;".to_string(), + attach_str, + ]) + } + _ => Err(Error::ExecutionErr(format!( + "Unsupported db type in DuckDB ATTACH: {}", + parsed.db_type + ))), + } +} + +// Returns the transformed query and the set of storages used +async fn transform_s3_uris( + query: &str, + client: &AuthedClient, +) -> Result<( + Option, + HashMap, DuckdbConnectionSettingsResponse>, +)> { + let mut transformed_query = None; + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap(); + } + let mut used_storages = HashMap::new(); + for cap in RE.captures_iter(query) { + if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) { + let s3_path = s3_path.as_str(); + let storage = match storage.map(|m| m.as_str()) { + Some("") | None => None, + Some(s) => Some(s.to_string()), + }; + let original_str_lit = + format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path); + let duckdb_conn_settings = client + .get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 { + s3_resource_path: None, + storage: storage.clone(), + }) + .await?; + let url = match &duckdb_conn_settings { + DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => { + format!("'s3://{bucket}/{s3_path}'") + } + DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => { + format!("'{base}/{s3_path}'") + } + _ => { + return Err(Error::ExecutionErr( + "DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(), + ))?; + } + }; + transformed_query = Some( + transformed_query + .unwrap_or(query.to_string()) + .replace(&original_str_lit, &url), + ); + used_storages.insert(storage, duckdb_conn_settings); + } + } + Ok((transformed_query, used_storages)) +} + +// BigQuery extension requires a json file as credentials +// The file path is set as an env var by do_duckdb +// It is created by transform_attach_db_resource_query (when bigquery is detected) +// and deleted by do_duckdb after the query is executed +fn make_bq_credentials_path(job_id: &Uuid) -> String { + format!("/tmp/service-account-credentials-{}.json", job_id) +} + +// duckdb-rs does not support named parameters, +// and it raises an error when passing unused arguments. We cannot prepare batch statements +// but only single SQL statements so it doesn't work when all arguments are not used by +// every single statement. +fn interpolate_named_args<'a>( + query: &str, + args: &'a HashMap, +) -> (String, Vec<&'a duckdb::types::Value>) { + let mut query = query.to_string(); + + let mut values = vec![]; + for (arg_name, arg_value) in args { + let pat = format!("${}", arg_name); + if !query.contains(&pat) { + continue; + } + values.push(arg_value); + query = query.replace(&pat, &format!("${}", values.len())); + } + (query, values) +} + +fn trunc_sig(query: &str) -> &str { + let idx = query.rfind("-- $").unwrap_or(query.len()); + // find next \n starting from idx and return everything after it + let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0); + &query[idx..] +} diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 4ea56c090e..fa6eb10960 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -20,6 +20,8 @@ mod csharp_executor; #[cfg(feature = "enterprise")] mod dedicated_worker; mod deno_executor; +#[cfg(feature = "duckdb")] +mod duckdb_executor; mod global_cache; mod go_executor; mod graphql_executor; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index ffd84db175..04403e4e08 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -32,13 +32,13 @@ use crate::{ }; #[derive(Deserialize)] -struct MysqlDatabase { - host: String, - user: Option, - password: Option, - port: Option, - database: String, - ssl: Option, +pub struct MysqlDatabase { + pub host: String, + pub user: Option, + pub password: Option, + pub port: Option, + pub database: String, + pub ssl: Option, } fn do_mysql_inner<'a>( diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 67c2d945aa..2c7fe123dc 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -47,14 +47,14 @@ use lazy_static::lazy_static; use urlencoding::encode; use windmill_common::client::AuthedClient; #[derive(Deserialize)] -struct PgDatabase { - host: String, - user: Option, - password: Option, - port: Option, - sslmode: Option, - dbname: String, - root_certificate_pem: Option, +pub struct PgDatabase { + pub host: String, + pub user: Option, + pub password: Option, + pub port: Option, + pub sslmode: Option, + pub dbname: String, + pub root_certificate_pem: Option, } lazy_static! { diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 3ae15e7453..1ecf28e664 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -2998,6 +2998,8 @@ var $RawScript = { "mssql", "graphql", "nativets", + "duckdb", + // for related places search: ADD_NEW_LANG ], }, path: { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 221650389a..e15c24ef70 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -145,6 +145,9 @@ use crate::ansible_executor::handle_ansible_job; #[cfg(feature = "mysql")] use crate::mysql_executor::do_mysql; +#[cfg(feature = "duckdb")] +use crate::duckdb_executor::do_duckdb; + #[cfg(feature = "oracledb")] use crate::oracledb_executor::do_oracledb; @@ -2614,6 +2617,30 @@ async fn handle_code_execution_job( ) .await; } + } else if language == Some(ScriptLang::DuckDb) { + #[allow(unreachable_code)] + #[cfg(not(feature = "duckdb"))] + { + return Err(Error::internal_err( + "Duck DB requires the duckdb feature to be enabled".to_string(), + )); + } + + #[cfg(feature = "duckdb")] + { + return do_duckdb( + job, + &client, + &code, + conn, + mem_peak, + canceled_by, + worker_name, + column_order, + occupancy_metrics, + ) + .await; + } } else if language == Some(ScriptLang::Graphql) { return do_graphql( job, @@ -3021,6 +3048,7 @@ fn parse_sig_of_lang( ScriptLang::Snowflake => Some(windmill_parser_sql::parse_snowflake_sig(code)?), ScriptLang::Graphql => None, ScriptLang::Mssql => Some(windmill_parser_sql::parse_mssql_sig(code)?), + ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?), ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?), #[cfg(feature = "php")] ScriptLang::Php => Some(windmill_parser_php::parse_php_signature( diff --git a/cli/bootstrap/script_bootstrap.ts b/cli/bootstrap/script_bootstrap.ts index 2ac67bdb83..e9e8a12db5 100644 --- a/cli/bootstrap/script_bootstrap.ts +++ b/cli/bootstrap/script_bootstrap.ts @@ -76,6 +76,7 @@ func main() (interface{}, error) { bash: `echo "Hello world" `, + duckdb: `SELECT 'Hello world' AS message`, oracledb: `SELECT 'Hello world' AS message`, powershell: `Write-Output "Hello world"`, @@ -126,5 +127,5 @@ public class Main { } } `, -// for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG }; diff --git a/cli/build.sh b/cli/build.sh index bf00b99a57..e228750b51 100755 --- a/cli/build.sh +++ b/cli/build.sh @@ -1,3 +1,4 @@ #!/bin/bash +# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`. ./gen_wm_client.sh deno run -A dnt.ts diff --git a/cli/metadata.ts b/cli/metadata.ts index 3a83143182..ee39d0b9da 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -528,6 +528,9 @@ export async function inferSchema( { name: "database", typ: { resource: "postgresql" } }, ...inferedSchema.args, ]; + } else if (language === "duckdb") { + const { parse_sql } = await import("./wasm/regex/windmill_parser_wasm.js"); + inferedSchema = JSON.parse(parse_sql(content)); } else if (language === "graphql") { const { parse_graphql } = await import( "./wasm/regex/windmill_parser_wasm.js" diff --git a/cli/script.ts b/cli/script.ts index 3896d19c5c..e12c15d9ba 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -547,6 +547,8 @@ export function filePathExtensionFromContentType( return ".my.sql"; } else if (language === "bigquery") { return ".bq.sql"; + } else if (language === "duckdb") { + return ".duckdb.sql"; } else if (language === "oracledb") { return ".odb.sql"; } else if (language === "snowflake") { @@ -573,7 +575,7 @@ export function filePathExtensionFromContentType( return ".nu"; } else if (language === "java") { return ".java"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -593,6 +595,7 @@ export const exts = [ ".odb.sql", ".sf.sql", ".ms.sql", + ".duckdb.sql", ".sql", ".gql", ".ps1", @@ -601,9 +604,8 @@ export const exts = [ ".cs", ".nu", ".playbook.yml", - ".java" - // for related places search: ADD_NEW_LANG - + ".java", + // for related places search: ADD_NEW_LANG ]; export function removeExtensionToPath(path: string): string { diff --git a/cli/script_common.ts b/cli/script_common.ts index a41d7dee8c..39906e7bb9 100644 --- a/cli/script_common.ts +++ b/cli/script_common.ts @@ -9,6 +9,7 @@ export type ScriptLanguage = | "postgresql" | "mysql" | "bigquery" + | "duckdb" | "oracledb" | "snowflake" | "mssql" @@ -19,7 +20,7 @@ export type ScriptLanguage = | "nu" | "ansible" | "java"; - // for related places search: ADD_NEW_LANG +// for related places search: ADD_NEW_LANG export function inferContentTypeFromFilePath( contentPath: string, @@ -43,6 +44,8 @@ export function inferContentTypeFromFilePath( return "bigquery"; } else if (contentPath.endsWith(".odb.sql")) { return "oracledb"; + } else if (contentPath.endsWith(".duckdb.sql")) { + return "duckdb"; } else if (contentPath.endsWith(".sf.sql")) { return "snowflake"; } else if (contentPath.endsWith(".ms.sql")) { @@ -67,7 +70,7 @@ export function inferContentTypeFromFilePath( return "nu"; } else if (contentPath.endsWith(".java")) { return "java"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error( "Invalid language: " + contentPath.substring(contentPath.lastIndexOf(".")) diff --git a/cli/sync.ts b/cli/sync.ts index 867b8c4d17..456afcadf9 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -344,6 +344,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { else if (language == "nu") ext = "nu"; else if (language == "ansible") ext = "playbook.yml"; else if (language == "java") ext = "java"; + else if (language == "duckdb") ext = "duckdb.sql"; // for related places search: ADD_NEW_LANG else ext = "no_ext"; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index 5418fad879..7867ab4a35 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -1,5 +1,6 @@ + let WASM_VECTOR_LEN = 0; let cachedUint8ArrayMemory0 = null; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 899306ebf0..8f93204d54 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -74,7 +74,7 @@ "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", - "windmill-parser-wasm-regex": "^1.481.0", + "windmill-parser-wasm-regex": "^1.492.1", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", @@ -12837,9 +12837,9 @@ "integrity": "sha512-EY3mSMWpqFPzd7fsLg2/hSfQFU8HpW9nplFwm4JHHCDbcTpBzlvzjPJoHAAGO5kMzowAxjqi5ai/mXjeUWuiSg==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.481.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.481.0.tgz", - "integrity": "sha512-QY9NOaF0M55SBEbbcAkHoXppjdS2gb2NlwqhlZ1ciYWTtIrbZoL5K8xoG4FbB5vWRHxInG1aRdA2tb+JP180yA==" + "version": "1.492.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.492.1.tgz", + "integrity": "sha512-CBdjz3x00z2xA4BRZv8/hsuHcpZgL28UKTjYVSj5O9XMYlY9g0LNsEQ5TTl/oVyFBs62/CPyvWwGvW4oSAObzg==" }, "node_modules/windmill-parser-wasm-rust": { "version": "1.429.0", diff --git a/frontend/package.json b/frontend/package.json index f302f5c873..9d3bbe19cc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -143,7 +143,7 @@ "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.477.1", - "windmill-parser-wasm-regex": "^1.481.0", + "windmill-parser-wasm-regex": "^1.492.1", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 35c4576251..02b7f4170b 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -133,6 +133,7 @@ import EditorTheme from './EditorTheme.svelte' import { BIGQUERY_TYPES, + DUCKDB_TYPES, MSSQL_TYPES, MYSQL_TYPES, ORACLEDB_TYPES, @@ -482,7 +483,9 @@ ? MSSQL_TYPES : scriptLang === 'oracledb' ? ORACLEDB_TYPES - : [] + : scriptLang === 'duckdb' + ? DUCKDB_TYPES + : [] ).map((t) => ({ label: t, kind: languages.CompletionItemKind.Function, diff --git a/frontend/src/lib/components/HighlightCode.svelte b/frontend/src/lib/components/HighlightCode.svelte index ac4357e5ca..bf196c7868 100644 --- a/frontend/src/lib/components/HighlightCode.svelte +++ b/frontend/src/lib/components/HighlightCode.svelte @@ -55,6 +55,8 @@ return sql case 'oracledb': return sql + case 'duckdb': + return sql case 'powershell': return powershell case 'php': @@ -66,10 +68,10 @@ case 'nu': return python case 'ansible': - return yaml; + return yaml case 'java': - return java; - // for related places search: ADD_NEW_LANG + return java + // for related places search: ADD_NEW_LANG default: return typescript } diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index 9dd3c9860a..6a3de43bf9 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -22,6 +22,7 @@ import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte' import NuIcon from '$lib/components/icons/NuIcon.svelte' import JavaIcon from '$lib/components/icons/JavaIcon.svelte' + import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' export let lang: | SupportedLanguage @@ -49,6 +50,7 @@ postgresql: 'Postgresql', bigquery: 'BigQuery', oracledb: 'Oracle Database', + duckdb: 'DuckDB', snowflake: 'Snowflake', mysql: 'MySQL', mssql: 'MS SQL Server', @@ -59,7 +61,7 @@ csharp: 'C#', nu: 'Nu', java: 'Java' - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } const langToComponent: Record< @@ -91,8 +93,9 @@ ansible: AnsibleIcon, csharp: CSharpIcon, nu: NuIcon, - java: JavaIcon - // for related places search: ADD_NEW_LANG + java: JavaIcon, + duckdb: DuckDbIcon + // for related places search: ADD_NEW_LANG } let subIconScale = width === 30 ? 0.6 : 0.8 diff --git a/frontend/src/lib/components/icons/DuckDbIcon.svelte b/frontend/src/lib/components/icons/DuckDbIcon.svelte new file mode 100644 index 0000000000..7229a377a6 --- /dev/null +++ b/frontend/src/lib/components/icons/DuckDbIcon.svelte @@ -0,0 +1,19 @@ + + + + + + + diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index d319868795..847bbf42da 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -203,6 +203,56 @@ export const MSSQL_TYPES = [ 'decimal', 'bit' ] +export const DUCKDB_TYPES = [ + 'BIGINT', + 'INT8', + 'LONG', + 'BIT', + 'BITSTRING', + 'BLOB', + 'BYTEA', + 'BINARY', + 'VARBINARY', + 'BOOLEAN', + 'BOOL', + 'LOGICAL', + 'DATE', + 'DECIMAL', + 'NUMERIC', + 'DOUBLE', + 'FLOAT8', + 'FLOAT', + 'FLOAT4', + 'REAL', + 'HUGEINT', + 'INTEGER', + 'INT4', + 'INT', + 'SIGNED', + 'INTERVAL', + 'JSON', + 'SMALLINT', + 'INT2', + 'SHORT', + 'TIME', + 'TIMESTAMP WITH TIME ZONE', + 'TIMESTAMPTZ', + 'TIMESTAMP', + 'DATETIME', + 'TINYINT', + 'INT1', + 'UBIGINT', + 'UHUGEINT', + 'UINTEGER', + 'USMALLINT', + 'UTINYINT', + 'UUID', + 'VARCHAR', + 'CHAR', + 'BPCHAR', + 'TEXT', + 'STRING' +] export const DB_TYPES: Record = { bigquery: BIGQUERY_TYPES, diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 5dcb6772f3..6582930715 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -9,6 +9,7 @@ import initRegexParsers, { parse_sql, parse_mysql, parse_oracledb, + parse_duckdb, parse_bigquery, parse_snowflake, parse_graphql, @@ -96,7 +97,9 @@ export async function inferArgs( } let inlineDBResource: string | undefined = undefined - if (['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '')) { + if ( + ['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '') + ) { await initWasmRegex() inlineDBResource = parse_db_resource(code) } @@ -147,6 +150,9 @@ export async function inferArgs( ...inferedSchema.args ] } + } else if (language == 'duckdb') { + await initWasmRegex() + inferedSchema = JSON.parse(parse_duckdb(code)) } else if (language == 'snowflake') { inferedSchema = JSON.parse(parse_snowflake(code)) if (inlineDBResource === undefined) { @@ -194,7 +200,7 @@ export async function inferArgs( } else if (language == 'java') { await initWasmJava() inferedSchema = JSON.parse(parse_java(code)) - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { return null } diff --git a/frontend/src/lib/inferArgSig.ts b/frontend/src/lib/inferArgSig.ts index ae4ad83794..68d3fc814e 100644 --- a/frontend/src/lib/inferArgSig.ts +++ b/frontend/src/lib/inferArgSig.ts @@ -5,23 +5,21 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { + oneof: { label: string properties: { key: string; typ: any }[] - } - ] - }, + }[] + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: '' } diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index ceb6644c42..07c05974dc 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -307,6 +307,19 @@ INSERT INTO demo VALUES (@P1, @P2); UPDATE demo SET col2 = @P3 WHERE col2 = @P2; ` +const DUCKDB_INIT_CODE = `-- $friends_csv (s3object) +-- $name (text) = Ben + +ATTACH '$res:u/demo/amazed_postgresql' AS db (TYPE postgres); +CREATE TABLE IF NOT EXISTS db.public.friends (name text); + +INSERT INTO db.public.friends + SELECT name FROM read_csv($friends_csv); + +SELECT 'Hello ' || $name || ', you have ' || + (SELECT COUNT(*) FROM read_csv($friends_csv)) || ' new friends !'; +` + const GRAPHQL_INIT_CODE = `query($name4: String, $name2: Int, $name3: [String]) { demo(name1: $name1, name2: $name2, name3: $name3) { name1, @@ -1134,6 +1147,9 @@ export const INITIAL_CODE = { mssql: { script: MSSQL_INIT_CODE }, + duckdb: { + script: DUCKDB_INIT_CODE + }, graphql: { script: GRAPHQL_INIT_CODE }, @@ -1258,6 +1274,8 @@ export function initialCode( return INITIAL_CODE.mssql.script } else if (language == 'graphql') { return INITIAL_CODE.graphql.script + } else if (language == 'duckdb') { + return INITIAL_CODE.duckdb.script } else if (language == 'php') { return INITIAL_CODE.php.script } else if (language == 'rust') { diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index bc8a44b434..961ae72488 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -38,6 +38,8 @@ export function scriptLangToEditorLang( return 'sql' } else if (lang == 'mssql') { return 'sql' + } else if (lang == 'duckdb') { + return 'sql' } else if (lang == 'python3') { return 'python' } else if (lang == 'bash') { @@ -140,7 +142,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['csharp', 'C#'], ['docker', 'Docker'], ['nu', 'Nu'], - ['java', 'Java'] + ['java', 'Java'], + ['duckdb', 'DuckDB'] // for related places search: ADD_NEW_LANG ] export function processLangs(selected: string | undefined, langs: string[]): string[] { @@ -150,7 +153,7 @@ export function processLangs(selected: string | undefined, langs: string[]): str let ls = langs.filter((lang) => lang !== 'nativets') //those languages are newer and may not be in the saved list - let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java'] + let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb'] // for related places search: ADD_NEW_LANG nl.forEach((lang) => { if (!ls.includes(lang)) { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 35f23fc9e7..cab4ef6adb 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -755,7 +755,7 @@ priority: {job.priority} {/if} - {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)} + {#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'dependency'].includes(job.tag)}
Tag: {job.tag} From feae9b09240ba306c007013a36d2aefb0b273766 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 27 May 2025 21:04:24 +0200 Subject: [PATCH 02/33] fix: error handler node rendering at top level --- frontend/src/lib/components/graph/graphBuilder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/graph/graphBuilder.ts b/frontend/src/lib/components/graph/graphBuilder.ts index 0011106ade..ab851db42e 100644 --- a/frontend/src/lib/components/graph/graphBuilder.ts +++ b/frontend/src/lib/components/graph/graphBuilder.ts @@ -166,7 +166,7 @@ export function graphBuilder( disableMoveIds: options?.disableMoveIds, enableTrigger: sourceId === 'Input', // If the index is -1, it means that the target module is not in the modules array, so we set it to the length of the array - index: index >= 0 ? index : mods?.length ?? 0, + index: index >= 0 ? index : (mods?.length ?? 0), ...extra, insertable: extra.insertable && !options?.disableInsert && prefix == undefined } @@ -635,7 +635,7 @@ export function graphBuilder( if (failureModule) { let toAdd: Record = {} Object.keys(extra.flowModuleStates ?? {}).forEach((id) => { - if (id.startsWith('failure-')) { + if (id.startsWith('failure')) { const failureState = extra.flowModuleStates?.[id] as GraphModuleState | undefined if (failureState?.parent_module) { toAdd[failureState.parent_module] = id From 0d459d5d223728270854e37715ecc1663ede9870 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 28 May 2025 01:21:22 +0200 Subject: [PATCH 03/33] fix: avoid monaco memory leak --- frontend/src/lib/components/Editor.svelte | 5 ++- .../src/lib/components/SimpleEditor.svelte | 31 +++++++++++------ .../src/lib/components/TemplateEditor.svelte | 32 +++++++++-------- .../copilot/autocomplete/monaco-adapter.ts | 34 +++++++++---------- frontend/src/lib/components/vscode.ts | 9 ++++- 5 files changed, 68 insertions(+), 43 deletions(-) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 02b7f4170b..aa1aa5f0b0 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -115,7 +115,7 @@ import { workspaceStore } from '$lib/stores' import { type Preview, ResourceService, UserService } from '$lib/gen' import type { Text } from 'yjs' - import { initializeVscode } from '$lib/components/vscode' + import { initializeVscode, keepModelAroundToAvoidDisposalOfWorkers } from '$lib/components/vscode' import { initializeMode } from 'monaco-graphql/esm/initializeMode.js' import type { MonacoGraphQLAPI } from 'monaco-graphql/esm/api.js' @@ -1266,6 +1266,8 @@ folding }) + keepModelAroundToAvoidDisposalOfWorkers() + // updateEditorKeybindingsMode(editor, 'vim', undefined) let ataModel: NodeJS.Timeout | undefined = undefined @@ -1355,6 +1357,7 @@ try { closeWebsockets() vimDisposable?.dispose() + console.log('disposing editor', editor, model) model?.dispose() editor && editor.dispose() console.log('disposed editor') diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 984b05564a..d3c498054d 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -7,6 +7,16 @@ import '@codingame/monaco-vscode-standalone-css-language-features' import '@codingame/monaco-vscode-standalone-typescript-language-features' import '@codingame/monaco-vscode-standalone-html-language-features' + import { + editor as meditor, + KeyCode, + KeyMod, + Uri as mUri, + languages, + type IRange, + type IDisposable + } from 'monaco-editor' + languages.typescript.javascriptDefaults.setCompilerOptions({ target: languages.typescript.ScriptTarget.Latest, allowNonTsExtensions: true, @@ -44,15 +54,15 @@ import { BROWSER } from 'esm-env' import { createHash, editorConfig, langToExt, updateOptions } from '$lib/editorUtils' - import { - editor as meditor, - KeyCode, - KeyMod, - Uri as mUri, - languages, - type IRange, - type IDisposable - } from 'monaco-editor' + // import { + // editor as meditor, + // KeyCode, + // KeyMod, + // Uri as mUri, + // languages, + // type IRange, + // type IDisposable + // } from 'monaco-editor' import { allClasses } from './apps/editor/componentsPanel/cssUtils' @@ -60,7 +70,7 @@ import libStdContent from '$lib/es6.d.ts.txt?raw' import domContent from '$lib/dom.d.ts.txt?raw' - import { initializeVscode } from './vscode' + import { initializeVscode, keepModelAroundToAvoidDisposalOfWorkers } from './vscode' import EditorTheme from './EditorTheme.svelte' import { vimMode } from '$lib/stores' import { initVim } from './monaco_keybindings' @@ -348,6 +358,7 @@ snippetsPreventQuickSuggestions: disableSuggestions } }) + keepModelAroundToAvoidDisposalOfWorkers() let timeoutModel: NodeJS.Timeout | undefined = undefined editor.onDidChangeModelContent((event) => { diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index 941c72fc74..22d3fc6776 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -1,3 +1,21 @@ + + + + { + confirmationModalOpen = false + }} + on:confirmed={() => { + onDelete?.() + confirmationModalOpen = false + }} +> + {#if trigger !== undefined} + {@const IconComponent = triggerIconMap[trigger.type]} +
+ + +
+ {/if} + + + - ``` - - - Use `$derived` for computed values based on other reactive state. - - ```svelte - - -

{count} * 2 = {doubled}

- ``` - - - Use `$effect` for side effects that need to run when reactive values change (e.g., logging, manual DOM manipulation, data fetching). Remember `$effect` does not run on the server. - - ```svelte - - ``` - -2. **Props with `$props`**: - - - Declare component props using `$props()`. This offers better clarity and flexibility compared to `export let`. - - ```svelte - - -

Name: {name}

-

Age: {age}

- ``` - - - For bindable props, use `$bindable`. - - ```svelte - - - - ``` - -## Event Handling - -- **Use direct event attributes**: Svelte 5 moves away from `on:` directives for DOM events. - - **Do**: `` - - **Don't**: `` -- **For component events, prefer callback props**: Instead of `createEventDispatcher`, pass functions as props. - - ```svelte - - - -

Message from child: {message}

- - - - - ``` - -## Snippets for Content Projection - -- **Use `{#snippet ...}` and `{@render ...}` instead of slots**: Snippets are more powerful and flexible. - - ```svelte - - - - - {#snippet title()} - My Awesome Title - {/snippet} - {#snippet content()} -

Some interesting content here.

- {/snippet} - - - - - -
-
{@render title()}
-
{@render content()}
-
- ``` - -- Default content is passed via the `children` prop (which is a snippet). - ```svelte - - -
- {@render children?.()} -
- ``` - -## Component Design - -1. **Create Small, Reusable Components**: Break down complex UIs into smaller, focused components. Each component should have a single responsibility. This also aids performance by limiting the scope of reactivity updates. -2. **Descriptive Naming**: Use clear and descriptive names for variables, functions, and components. -3. **Minimize Logic in Components**: Move complex business logic to utility functions or services. Keep components focused on presentation and interaction. - -## State Management (Stores) - -1. **Segment Stores**: Avoid a single global store. Create multiple stores, each responsible for a specific piece of global state (e.g., `userStore.js`, `themeStore.js`). This can help limit reactivity updates to only the parts of the UI that depend on specific state segments. -2. **Use Custom Stores for Complex Logic**: For stores with related methods, create custom stores. - - ```javascript - // counterStore.js - import { writable } from 'svelte/store' - - function createCounter() { - const { subscribe, set, update } = writable(0) - - return { - subscribe, - increment: () => update((n) => n + 1), - decrement: () => update((n) => n - 1), - reset: () => set(0) - } - } - export const counter = createCounter() - ``` - -3. **Use Context API for Localized State**: For state shared within a component subtree, consider Svelte's context API (`setContext`, `getContext`) instead of global stores when the state doesn't need to be truly global. - -## Performance Optimizations (Svelte 5) - -When generating Svelte 5 code, prioritize frontend performance by applying the following principles: - -### General Svelte 5 Principles - -- **Leverage the Compiler:** Trust Svelte's compiler to generate optimized JavaScript. Avoid manual DOM manipulation (`document.querySelector`, etc.) unless absolutely necessary for integrating third-party libraries that lack Svelte adapters. -- **Keep Components Small and Focused:** Reinforcing from Component Design, smaller components lead to less complex reactivity graphs and more targeted, efficient updates. - -### Reactivity & State Management - -- **Optimize Computations with `$derived`:** Always use `$derived` for computed values that depend on other state. This ensures the computation only runs when its specific dependencies change, avoiding unnecessary work compared to recomputing derived values in `$effect` or less efficient methods. -- **Minimize `$effect` Usage:** Use `$effect` sparingly and only for true side effects that interact with the outside world or non-Svelte state. Avoid putting complex logic or state updates _within_ an `$effect` unless those updates are explicitly intended as a reaction to external changes or non-Svelte state. Excessive or complex effects can impact rendering performance. -- **Structure State for Fine-Grained Updates:** Design your `$state` objects or variables such that updates affect only the necessary parts of the UI. Avoid putting too much unrelated state into a single large object that gets frequently updated, as this can potentially trigger broader updates than necessary. Consider normalizing complex, nested state. - -### List Rendering (`{#each}`) - -- **Mandate `key` Attribute:** Always use a `key` attribute (`{#each items as item (item.id)}`) that refers to a unique, stable identifier for each item in a list. This is critical for allowing Svelte to efficiently update, reorder, add, or remove list items without destroying and re-creating unnecessary DOM elements and component instances. - -### Component Loading & Bundling - -- **Implement Lazy Loading/Code Splitting:** For routes, components, or modules that are not immediately needed on page load, use dynamic imports (`import(...)`) to split the code bundle. SvelteKit handles this automatically for routes, but it can be applied manually to components using helper patterns if needed. -- **Be Mindful of Third-Party Libraries:** When incorporating external libraries, import only the necessary functions or components to minimize the final bundle size. Prefer libraries designed to be tree-shakeable. - -### Rendering & DOM - -- **Use CSS for Animations/Transitions:** Prefer CSS animations or transitions where possible for performance. Svelte's built-in `transition:` directive is also highly optimized and should be used for complex state-driven transitions, but simple cases can often use plain CSS. -- **Optimize Image Loading:** Implement best practices for images: use optimized formats (WebP, AVIF), lazy loading (`loading="lazy"`), and responsive images (``, `srcset`) to avoid loading unnecessarily large images. - -### Server-Side Rendering (SSR) & Hydration - -- **Ensure SSR Compatibility:** Write components that can be rendered on the server for faster initial page loads. Avoid relying on browser-specific APIs (like `window` or `document`) in the main ` - -
- Hello -
- ``` - -5. **Stay Updated**: Keep Svelte and its related packages up to date to benefit from the latest features, performance improvements, and security fixes. From e302aa38b5977dd406ae05e1d8dbb74cb7dc3d17 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 May 2025 01:16:50 +0200 Subject: [PATCH 13/33] fix: evalv2 prohibit component delete --- .../components/apps/editor/AppEditor.svelte | 3 +- .../apps/editor/SettingsPanel.svelte | 12 +++- .../inputEditor/EvalV2InputEditor.svelte | 56 +++++++++++++------ 3 files changed, 49 insertions(+), 22 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index fd328beb28..73a5b5b70b 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -785,8 +785,7 @@ $: forceDeactivatePanzoom = isModifierKeyPressed && handMode - - + diff --git a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte index b2e0a4b52e..108a2498c2 100644 --- a/frontend/src/lib/components/apps/editor/SettingsPanel.svelte +++ b/frontend/src/lib/components/apps/editor/SettingsPanel.svelte @@ -121,9 +121,15 @@ gridItemWithLocation?.location.type === 'subgrid' && Array.isArray($app.subgrids?.[gridItemWithLocation.location.subgridKey]) ) { - $app.subgrids[gridItemWithLocation.location.subgridKey][ - gridItemWithLocation.location.subgridItemIndex - ] = cs.item + if ( + $app.subgrids[gridItemWithLocation.location.subgridKey][ + gridItemWithLocation.location.subgridItemIndex + ] + ) { + $app.subgrids[gridItemWithLocation.location.subgridKey][ + gridItemWithLocation.location.subgridItemIndex + ] = cs.item + } } } } diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/EvalV2InputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/EvalV2InputEditor.svelte index 3b59f74051..dd9c7aebb6 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/EvalV2InputEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/EvalV2InputEditor.svelte @@ -13,14 +13,27 @@ import { zIndexes } from '$lib/zIndexes' import Popover from '$lib/components/Popover.svelte' - export let componentInput: EvalV2AppInput | undefined - export let id: string - export let field: string - export let fixedOverflowWidgets: boolean = true - export let acceptSelf: boolean = false - export let recomputeOnInputChanged = true - export let showOnDemandOnlyToggle = false - export let securedContext = false + interface Props { + componentInput: EvalV2AppInput | undefined + id: string + field: string + fixedOverflowWidgets?: boolean + acceptSelf?: boolean + recomputeOnInputChanged?: boolean + showOnDemandOnlyToggle?: boolean + securedContext?: boolean + } + + let { + componentInput = $bindable(), + id, + field, + fixedOverflowWidgets = true, + acceptSelf = false, + recomputeOnInputChanged = true, + showOnDemandOnlyToggle = false, + securedContext = false + }: Props = $props() const { onchange, @@ -30,15 +43,17 @@ } = getContext('AppViewerContext') const { evalPreview } = getContext('AppEditorContext') - let editor: SimpleEditor + let editor: SimpleEditor | undefined = $state() export function setCode(code: string) { editor?.setCode(code) } - $: extraLib = - componentInput?.expr && $worldStore + let exprDefined = $derived(componentInput?.expr != undefined) + let extraLib = $derived( + exprDefined && $worldStore ? buildExtraLib($worldStore?.outputsById ?? {}, acceptSelf ? '' : id, $stateStore, false) : undefined + ) if ( componentInput && @@ -58,8 +73,8 @@ } } - let fullscreen = false - let focus = false + let fullscreen = $state(false) + let focus = $state(false) {#if componentInput?.type === 'evalv2'} @@ -110,7 +125,14 @@ small bind:this={editor} lang="javascript" - bind:code={() => componentInput.expr ?? '', (e) => (componentInput.expr = e)} + bind:code={ + () => componentInput.expr ?? '', + (e) => { + if (componentInput.expr != e) { + componentInput.expr = e + } + } + } shouldBindKey={false} {extraLib} autoHeight @@ -132,17 +154,17 @@
- + {#snippet text()} This context variable is securely provided by the backend and cannot be altered by users - + {/snippet}
{/if} (fullscreen = true)}> {#if focus}
From 8d12bcc8ee2991909ea0d9bb57f04f0d4106c69f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 May 2025 02:10:17 +0200 Subject: [PATCH 14/33] fix: faster layout for larger graphs --- .../lib/components/graph/FlowGraphV2.svelte | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 6fa84591f2..f0c1b0bc9e 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -25,7 +25,7 @@ import ResultNode from './renderers/nodes/ResultNode.svelte' import BaseEdge from './renderers/edges/BaseEdge.svelte' import EmptyEdge from './renderers/edges/EmptyEdge.svelte' - import { sugiyama, dagStratify, decrossOpt, coordCenter } from 'd3-dag' + import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' import { Expand } from 'lucide-svelte' import Toggle from '../Toggle.svelte' import DataflowEdge from './renderers/edges/DataflowEdge.svelte' @@ -117,7 +117,7 @@ function onModulesChange(modules: FlowModule[]) { computeSimplifiableFlow( modules, - triggerContext?.simplifiedPoll ? get(triggerContext.simplifiedPoll) ?? false : false + triggerContext?.simplifiedPoll ? (get(triggerContext.simplifiedPoll) ?? false) : false ) } @@ -143,12 +143,13 @@ let boxSize: any try { const layout = sugiyama() - .decross(decrossOpt()) + .decross(nodes.length > 20 ? decrossTwoLayer() : decrossOpt()) .coord(coordCenter()) .nodeSize(() => [NODE.width + NODE.gap.horizontal, NODE.height + NODE.gap.vertical]) boxSize = layout(dag) } catch { const layout = sugiyama() + .decross(decrossTwoLayer()) .coord(coordCenter()) .nodeSize(() => [NODE.width + NODE.gap.horizontal, NODE.height + NODE.gap.vertical]) boxSize = layout(dag) @@ -160,13 +161,13 @@ position: { x: des.x ? // @ts-ignore - (des.data.data.offset ?? 0) + - // @ts-ignore - des.x + - (fullSize ? fullWidth : width) / 2 - - boxSize.width / 2 - - NODE.width / 2 - - (width - fullWidth) / 2 + (des.data.data.offset ?? 0) + + // @ts-ignore + des.x + + (fullSize ? fullWidth : width) / 2 - + boxSize.width / 2 - + NODE.width / 2 - + (width - fullWidth) / 2 : 0, y: des.y || 0 } From eb33b3603e981b655c11b467cba7401d4a525300 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 May 2025 02:28:32 +0200 Subject: [PATCH 15/33] chore(main): release 1.493.3 (#5833) * chore(main): release 1.493.3 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 8 ++ backend/Cargo.lock | 84 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 67 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bdd6a9a9f..82eaaa7ca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.493.3](https://github.com/windmill-labs/windmill/compare/v1.493.2...v1.493.3) (2025-05-29) + + +### Bug Fixes + +* evalv2 prohibit component delete ([e302aa3](https://github.com/windmill-labs/windmill/commit/e302aa38b5977dd406ae05e1d8dbb74cb7dc3d17)) +* faster layout for larger graphs ([8d12bcc](https://github.com/windmill-labs/windmill/commit/8d12bcc8ee2991909ea0d9bb57f04f0d4106c69f)) + ## [1.493.2](https://github.com/windmill-labs/windmill/compare/v1.493.1...v1.493.2) (2025-05-28) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 445dad0222..047fd36c74 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -791,7 +791,7 @@ dependencies = [ "bytes", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "serde-aux", "serde_json", @@ -5990,7 +5990,7 @@ dependencies = [ "google-cloud-token", "home", "jsonwebtoken 9.3.1", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "serde_json", "thiserror 1.0.69", @@ -6033,7 +6033,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d901aeb453fd80e51d64df4ee005014f6cf39f2d736dd64f7239c132d9d39a6a" dependencies = [ - "reqwest 0.12.17", + "reqwest 0.12.18", "thiserror 1.0.69", "tokio", ] @@ -8639,7 +8639,7 @@ dependencies = [ "getrandom 0.2.16", "http 1.3.1", "rand 0.8.5", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "serde_json", "serde_path_to_error", @@ -8688,7 +8688,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.37.5", "rand 0.9.0", - "reqwest 0.12.17", + "reqwest 0.12.18", "ring 0.17.14", "serde", "serde_json", @@ -10376,9 +10376,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.17" +version = "0.12.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3f27698fc5799d2a281528a908bd874973953ca2e7bc2311637e10c263c723b" +checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" dependencies = [ "async-compression", "base64 0.22.1", @@ -10434,7 +10434,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.3.1", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "thiserror 1.0.69", "tower-service", @@ -10453,7 +10453,7 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "parking_lot 0.11.2", - "reqwest 0.12.17", + "reqwest 0.12.18", "reqwest-middleware", "retry-policies", "thiserror 1.0.69", @@ -13671,7 +13671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3beec919fbdf99d719de8eda6adae3281f8a5b71ae40431f44dc7423053d34" dependencies = [ "loki-api", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "serde_json", "snap", @@ -14667,7 +14667,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "axum", @@ -14687,7 +14687,7 @@ dependencies = [ "prometheus", "quote", "rand 0.9.0", - "reqwest 0.12.17", + "reqwest 0.12.18", "rustls 0.23.27", "serde", "serde_json", @@ -14718,7 +14718,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "argon2", @@ -14777,7 +14777,7 @@ dependencies = [ "rand 0.9.0", "rdkafka", "regex", - "reqwest 0.12.17", + "reqwest 0.12.18", "rmcp", "rsa", "rumqttc", @@ -14827,7 +14827,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.493.2" +version = "1.493.3" dependencies = [ "base64 0.22.1", "chrono", @@ -14842,7 +14842,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.493.2" +version = "1.493.3" dependencies = [ "chrono", "serde", @@ -14855,7 +14855,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "serde", @@ -14869,7 +14869,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "async-stream", @@ -14911,7 +14911,7 @@ dependencies = [ "quick_cache", "rand 0.9.0", "regex", - "reqwest 0.12.17", + "reqwest 0.12.18", "reqwest-middleware", "reqwest-retry", "semver 1.0.26", @@ -14945,7 +14945,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.493.2" +version = "1.493.3" dependencies = [ "regex", "serde", @@ -14959,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "bytes", @@ -14982,7 +14982,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.493.2" +version = "1.493.3" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14994,7 +14994,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.493.2" +version = "1.493.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -15003,7 +15003,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "lazy_static", @@ -15015,7 +15015,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "serde_json", @@ -15027,7 +15027,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "gosyn", @@ -15039,7 +15039,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "lazy_static", @@ -15051,7 +15051,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "serde_json", @@ -15063,7 +15063,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "nu-parser", @@ -15074,7 +15074,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15085,7 +15085,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15096,7 +15096,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "async-recursion", @@ -15119,7 +15119,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "lazy_static", @@ -15148,7 +15148,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "lazy_static", @@ -15166,7 +15166,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15190,7 +15190,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "serde_json", @@ -15200,7 +15200,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "async-recursion", @@ -15217,7 +15217,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.17", + "reqwest 0.12.18", "serde", "serde_json", "serde_urlencoded", @@ -15233,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.493.2" +version = "1.493.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.493.2" +version = "1.493.3" dependencies = [ "anyhow", "async-recursion", @@ -15295,7 +15295,7 @@ dependencies = [ "prometheus", "rand 0.9.0", "regex", - "reqwest 0.12.17", + "reqwest 0.12.18", "reqwest-middleware", "rust_decimal", "serde", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ea651b94e0..9556113099 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.493.2" +version = "1.493.3" authors.workspace = true edition.workspace = true @@ -32,7 +32,7 @@ members = [ ] [workspace.package] -version = "1.493.2" +version = "1.493.3" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8055b2654b..7e859bb707 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.493.2 + version: 1.493.3 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 3ff3ab48d5..666b7df69f 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.493.2"; +export const VERSION = "v1.493.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index f2d53e07ff..47d4198869 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -63,7 +63,7 @@ export { // } // }); -export const VERSION = "1.493.2"; +export const VERSION = "1.493.3"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 71b24d6cfc..53e102080f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.493.2", + "version": "1.493.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.493.2", + "version": "1.493.3", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index e875005166..3fd50f727d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.493.2", + "version": "1.493.3", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index ee923c3f00..33c426f043 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.493.2" -wmill_pg = ">=1.493.2" +wmill = ">=1.493.3" +wmill_pg = ">=1.493.3" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index c5adfe40d4..7c92b0825b 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.493.2 + version: 1.493.3 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 3063cfdbf5..f2181c8da2 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.493.2' + ModuleVersion = '1.493.3' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index d9e6a43f4b..a0982fe3bb 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.493.2" +version = "1.493.3" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 8a15766a48..7dd3cfc7a6 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.493.2" +version = "1.493.3" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 8bc58c45c2..3a31249db0 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.493.2", + "version": "1.493.3", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index b6fc492db9..2461a1e07e 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.493.2", + "version": "1.493.3", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index b99c89f1ea..0312f63a12 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.493.2 +1.493.3 From ed3ad327a235c16b9f3aa7f8edeefe61b0c01da3 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 29 May 2025 03:34:05 -0400 Subject: [PATCH 16/33] fix: templatev2 delete issue (#5834) --- frontend/src/lib/components/TemplateEditor.svelte | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index 22d3fc6776..c76e016a2b 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -477,8 +477,11 @@ editor.onDidChangeModelContent((event) => { timeoutModel && clearTimeout(timeoutModel) timeoutModel = setTimeout(() => { - code = getCode() - dispatch('change', { code }) + const newCode = getCode() + if (code != newCode) { + code = newCode + } + dispatch('change', { code: newCode }) }, 200) }) @@ -504,7 +507,10 @@ editor.onDidBlurEditorText(() => { dispatch('blur') - code = getCode() + const newCode = getCode() + if (code != newCode) { + code = newCode + } }) jsLoader = setTimeout(async () => { From acc352b5c009778603e77912e454f6359a0cc23a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 May 2025 09:56:48 +0200 Subject: [PATCH 17/33] opt code updates --- frontend/src/lib/components/Editor.svelte | 52 +++++++++++-------- .../src/lib/components/SimpleEditor.svelte | 31 +++++++---- .../src/lib/components/TemplateEditor.svelte | 27 +++++----- 3 files changed, 67 insertions(+), 43 deletions(-) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 8e573c766d..57bebf80a3 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -344,26 +344,36 @@ } export function setCode(ncode: string, noHistory: boolean = false): void { - code = ncode - if (noHistory) { - editor?.setValue(ncode) - } else { - if (editor?.getModel()) { - // editor.setValue(ncode) - editor.pushUndoStop() + if (code != ncode) { + if (noHistory) { + editor?.setValue(ncode) + } else { + if (editor?.getModel()) { + // editor.setValue(ncode) + editor.pushUndoStop() - editor.executeEdits('set', [ - { - range: editor.getModel()!.getFullModelRange(), // full range - text: ncode - } - ]) + editor.executeEdits('set', [ + { + range: editor.getModel()!.getFullModelRange(), // full range + text: ncode + } + ]) - editor.pushUndoStop() + editor.pushUndoStop() + } } } } + function updateCode(): boolean { + const ncode = getCode() + if (code == ncode) { + return false + } + code = ncode + return true + } + export function append(code: string): void { if (editor) { const lineCount = editor.getModel()?.getLineCount() || 0 @@ -387,7 +397,7 @@ export async function format() { if (editor) { - code = getCode() + updateCode() if (lang != 'shell' && lang != 'nu') { if ($formatOnSave != false) { if (scriptLang == 'deno' && languageClients.length > 0) { @@ -424,7 +434,7 @@ await editor?.getAction('editor.action.formatDocument')?.run() } } - code = getCode() + updateCode() } if (formatAction) { formatAction() @@ -1275,9 +1285,9 @@ editor?.onDidChangeModelContent((event) => { timeoutModel && clearTimeout(timeoutModel) timeoutModel = setTimeout(() => { - let ncode = getCode() - code = ncode - dispatch('change', ncode) + if (updateCode()) { + dispatch('change', code) + } }, changeTimeout) ataModel && clearTimeout(ataModel) @@ -1296,12 +1306,12 @@ dispatch('focus') editor?.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () { - code = getCode() + updateCode() shouldBindKey && format && format() }) editor?.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () { - code = getCode() + updateCode() shouldBindKey && cmdEnterAction && cmdEnterAction() }) diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index 7cca3416ae..91bc2be3c0 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -157,8 +157,19 @@ } export function setCode(ncode: string): void { + if (ncode != code) { + code = ncode + editor?.setValue(ncode) + } + } + + function updateCode(): boolean { + const ncode = getCode() + if (code == ncode) { + return false + } code = ncode - editor?.setValue(ncode) + return true } function updatePlaceholderVisibility(value: string) { @@ -171,11 +182,11 @@ export function format() { if (editor) { - code = getCode() + updateCode() editor.getAction('editor.action.formatDocument')?.run() if (formatAction) { formatAction() - code = getCode() + updateCode() } } } @@ -368,8 +379,9 @@ suggestion = '' timeoutModel && clearTimeout(timeoutModel) timeoutModel = setTimeout(() => { - code = getCode() - dispatch('change', { code }) + if (updateCode()) { + dispatch('change', { code }) + } }, 200) }) @@ -379,7 +391,7 @@ loadExtraLib() editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () { - code = getCode() + updateCode() shouldBindKey && format && format() }) @@ -411,12 +423,12 @@ editor.onDidFocusEditorText(() => { if (!editor) return editor.addCommand(KeyMod.CtrlCmd | KeyCode.KeyS, function () { - code = getCode() + updateCode() shouldBindKey && format && format() }) editor.addCommand(KeyMod.CtrlCmd | KeyCode.Enter, function () { - code = getCode() + updateCode() shouldBindKey && cmdEnterAction && cmdEnterAction() }) dispatch('focus') @@ -424,8 +436,7 @@ editor.onDidBlurEditorText(() => { dispatch('blur') - - code = getCode() + updateCode() }) if (lang === 'css' && !cssClassesLoaded) { diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index c76e016a2b..0dfdfedfab 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -380,7 +380,6 @@ $componentControl[$selectedComponent[0]] = { ...$componentControl[$selectedComponent[0]], setCode: (value: string) => { - code = value setCode(value) } } @@ -411,9 +410,9 @@ } export function setCode(ncode: string): void { - code = ncode - if (editor) { - editor.setValue(ncode) + if (code != ncode) { + code = ncode + editor?.setValue(ncode) } } @@ -473,15 +472,22 @@ editor.addCommand(KeyMod.CtrlCmd | KeyMod.Shift | KeyCode.Digit7, function () {}) }) + function updateCode(): boolean { + const ncode = getCode() + if (code == ncode) { + return false + } + code = ncode + return true + } + let timeoutModel: NodeJS.Timeout | undefined = undefined editor.onDidChangeModelContent((event) => { timeoutModel && clearTimeout(timeoutModel) timeoutModel = setTimeout(() => { - const newCode = getCode() - if (code != newCode) { - code = newCode + if (updateCode()) { + dispatch('change', { code }) } - dispatch('change', { code: newCode }) }, 200) }) @@ -507,10 +513,7 @@ editor.onDidBlurEditorText(() => { dispatch('blur') - const newCode = getCode() - if (code != newCode) { - code = newCode - } + updateCode() }) jsLoader = setTimeout(async () => { From 453e7f03d3c7f3deb3c1d53f5d18740f2a30844d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 29 May 2025 10:25:05 +0200 Subject: [PATCH 18/33] nit optimize updates --- .../propertyPicker/ObjectViewer.svelte | 179 ++++++++++-------- 1 file changed, 101 insertions(+), 78 deletions(-) diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index 74225bdf4a..c9f4fec930 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -1,4 +1,6 @@ +{#snippet renderScalar(k: string, v: any)} + + + { + selectProp(k, v, true) + }} + title={JSON.stringify(v)} + > + {#if v === NEVER_TESTED_THIS_FAR} + Test the flow to see a value + {:else if v == undefined} + undefined + {:else if v == null} + null + {:else if typeof v == 'string'} + "{truncate(v, 200)}" + {:else if typeof v == 'number' && Number.isInteger(v) && !Number.isSafeInteger(v)} + + {truncate(JSON.stringify(v), 200)} + + + {#snippet text()} + This number is too large for the frontend to handle correctly and may be rounded. + {/snippet} + + + {:else} + + {truncate(JSON.stringify(v), 200)} + + {/if} + +{/snippet} + @@ -118,10 +175,11 @@ {#if searchOpen}
{ - if (event?.key === 'Escape') { + onkeydown={(event) => { + if ((event as KeyboardEvent)?.key === 'Escape') { clearSearch() } + event.stopPropagation() }} type="text" class="!h-6 !text-2xs mt-0.5" @@ -130,7 +188,7 @@ /> @@ -151,8 +209,8 @@ {#if !fullyCollapsed} {#if level != 0 && keys.length > 1} - - + + {/if} @@ -275,7 +296,7 @@
+ {:else if itemsType?.type == 'object' && itemsType?.resourceType == 's3object'} +
+ { + value = [ + ...value, + { + s3: evt.detail?.path ?? '', + filename: evt.detail?.filename ?? '' + } + ] + }} + on:deletion={(evt) => { + value = value.filter((v) => v.s3 !== evt.detail?.path) + }} + defaultValue={defaultValue?.map((v) => v.s3)} + initialValue={value} + /> +
{:else}
{#key redraw} diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 89285414de..3694b021b7 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -21,9 +21,19 @@ } | undefined - let selected: 'string' | 'number' | 'object' | 'bytes' | 'enum' | 'resource' | undefined = + let selected: + | 'string' + | 'number' + | 'object' + | 'bytes' + | 'enum' + | 'resource' + | 's3object' + | undefined = itemsType?.type != 'string' - ? itemsType?.type + ? itemsType?.type == 'object' && itemsType?.resourceType == 's3object' + ? 's3object' + : itemsType?.type : Array.isArray(itemsType?.enum) ? 'enum' : itemsType?.contentEncoding == 'base64' @@ -62,6 +72,8 @@ itemsType = { type: 'string', contentEncoding: 'base64' } } else if (selected == 'resource') { itemsType = { type: 'resource', resourceType: itemsType?.resourceType } + } else if (selected == 's3object') { + itemsType = { type: 'object', resourceType: 's3object' } } else { itemsType = undefined } @@ -71,6 +83,7 @@ {#if originalType != 'string[]'} + diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index 0212f5f534..77d9b308d8 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -18,7 +18,7 @@ export let iconSize = 36 export let returnFileNames = false export let submittedText: string | undefined = undefined - export let defaultFile: string | undefined = undefined + export let defaultFile: string | string[] | undefined = undefined export let disabled: boolean | undefined = undefined export let folderOnly = false @@ -160,9 +160,8 @@ files = files if (convertTo && files) { const promises = files.map(convertFile) - let converted: ConvertedFile[] | { name: string; data: ConvertedFile }[] = await Promise.all( - promises - ) + let converted: ConvertedFile[] | { name: string; data: ConvertedFile }[] = + await Promise.all(promises) if (returnFileNames) { converted = converted.map((c, i) => ({ name: files![i].name, data: c })) } @@ -239,7 +238,7 @@ {multiple} {...$$restProps} /> - {#if defaultFile} + {#if defaultFile && (!Array.isArray(defaultFile) || defaultFile.length > 0)}
Default file: {defaultFile}
diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index 9e5180b76c..44f2a7d8d6 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -12,69 +12,111 @@ import { createEventDispatcher, onDestroy } from 'svelte' import { emptyString } from '$lib/utils' - export let acceptedFileTypes: string[] | undefined = ['*'] - export let allowMultiple: boolean = true - export let allowDelete = true - export let folderOnly = false - export let containerText: string = folderOnly - ? 'Drag and drop a folder here or click to browse' - : allowMultiple - ? 'Drag and drop files here or click to browse' - : 'Drag and drop a file here or click to browse' - export let customResourcePath: string | undefined = undefined - export let customResourceType: 's3' | 'azure_blob' | undefined = undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not - export let customClass: string = '' - export let customStyle: string = '' - export let randomFileKey: boolean = false - export let pathTransformer: any = undefined // function taking as input {file: File} and returning a string - export let forceDisplayUploads: boolean = false - export let defaultValue: string | undefined = undefined - export let workspace: string | undefined = undefined - export let fileUploads: Writable = writable([]) - export let appPath: string | undefined = undefined - export let disabled = false - export let iconSize: number | undefined = undefined - export let initialValue: - | { - s3: string - filename: string - } - | undefined = undefined - - init() - - function init() { - if (initialValue?.s3) { - if (!$fileUploads.find((fileUpload) => fileUpload.path === initialValue?.s3)) { - $fileUploads = [ - ...$fileUploads, - { - name: initialValue.filename, - size: 0, - progress: 100, - path: initialValue?.s3 - } - ] - } - } + interface Props { + acceptedFileTypes?: string[] | undefined + allowMultiple?: boolean + allowDelete?: boolean + folderOnly?: boolean + containerText?: string + customResourcePath?: string | undefined + customResourceType?: 's3' | 'azure_blob' | undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not + customClass?: string + customStyle?: string + randomFileKey?: boolean + pathTransformer?: any // function taking as input {file: File} and returning a string + forceDisplayUploads?: boolean + defaultValue?: string | string[] | undefined + workspace?: string | undefined + fileUploads?: Writable + appPath?: string | undefined + disabled?: boolean + iconSize?: number | undefined + initialValue?: + | { + s3: string + filename: string + } + | { + s3: string + filename: string + }[] + | undefined + computeForceViewerPolicies?: + | (() => + | { + allowed_resources: string[] + allow_user_resources: boolean + allow_workspace_resource: boolean + file_key_regex: string + } + | undefined) + | undefined } - export let computeForceViewerPolicies: - | (() => - | { - allowed_resources: string[] - allow_user_resources: boolean - allow_workspace_resource: boolean - file_key_regex: string - } - | undefined) - | undefined = undefined + let { + acceptedFileTypes = ['*'], + allowMultiple = true, + allowDelete = true, + folderOnly = false, + containerText = folderOnly + ? 'Drag and drop a folder here or click to browse' + : allowMultiple + ? 'Drag and drop files here or click to browse' + : 'Drag and drop a file here or click to browse', + customResourcePath = undefined, + customResourceType = undefined, + customClass = '', + customStyle = '', + randomFileKey = false, + pathTransformer = undefined, + forceDisplayUploads = $bindable(false), + defaultValue = undefined, + workspace = undefined, + fileUploads = writable([]), + appPath = undefined, + disabled = false, + iconSize = undefined, + initialValue = undefined, + computeForceViewerPolicies = undefined + }: Props = $props() const dispatch = createEventDispatcher<{ addition: { path?: string; filename?: string } deletion: { path: string } }>() + let initialS3 = $derived( + Array.isArray(initialValue) + ? initialValue?.map((v) => v.s3) + : initialValue?.s3 + ? [initialValue?.s3] + : undefined + ) + init() + + function init() { + if (initialS3) { + function transform(initialValue: { s3: string; filename: string }) { + return { + name: initialValue.filename, + size: 0, + progress: 100, + path: initialValue?.s3 + } + } + for (const s3 of initialS3) { + if (!$fileUploads.find((fileUpload) => fileUpload.path === s3)) { + let initialFileUploads = initialValue + ? Array.isArray(initialValue) + ? initialValue.map(transform) + : [transform(initialValue)] + : [] + $fileUploads = [...$fileUploads, ...initialFileUploads] + } + } + } + } + type FileUploadData = { name: string size: number @@ -554,10 +596,10 @@ {containerText}{#if disabled}
(Disabled){/if} {/if} - {#if initialValue?.s3 && $fileUploads.length == 0} + {#if initialS3 && initialS3.length > 0 && $fileUploads.length == 0}
- File currently selected: {initialValue?.s3} + File currently selected: {initialS3?.join(', ')}
{/if} diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index c9f4fec930..97d541a673 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -34,7 +34,7 @@ level = 0, currentPath = '', pureViewer = false, - collapsed = $bindable((level != 0 && level % 3 == 0) || Array.isArray(json)), + collapsed = $bindable((level != 0 && level % 3 == 0) || (Array.isArray(json) && level != 0)), rawKey = false, topBrackets = false, allowCopy = true, @@ -205,6 +205,7 @@ {/if}
{/if} + {#if keys.length > 0} {#if !fullyCollapsed} From 80277d14d02e8e596c7002326946142226d382a6 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 29 May 2025 12:52:00 -0400 Subject: [PATCH 27/33] fix: allow disable tabs for sidebar/accordion tabs (#5838) --- .../apps/components/layout/AppTabs.svelte | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/apps/components/layout/AppTabs.svelte b/frontend/src/lib/components/apps/components/layout/AppTabs.svelte index c033165c69..977ccc69d9 100644 --- a/frontend/src/lib/components/apps/components/layout/AppTabs.svelte +++ b/frontend/src/lib/components/apps/components/layout/AppTabs.svelte @@ -151,10 +151,11 @@ )} style={css?.tabRow?.style} > - {#each tabs ?? [] as res} + {#each tabs ?? [] as res, index}
+ +
+ {/if} {#if !emptyStringTrimmed(gcp_resource_path) && !emptyStringTrimmed(topic_id)}
{ @@ -119,6 +132,7 @@ iconOnly />