From 3d0ca92ef2ad98b1c3e5abe68a7a5b10c94f05df Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 2 Nov 2026 15:40:22 +0100 Subject: [PATCH] feat: upgrade the bundled DuckDB engine to 2.0.0 LTS (#11924) * feat: upgrade the bundled DuckDB engine to 2.0.0 LTS Co-Authored-By: Claude Opus 5.5 * fix: never repeat a ducklake attach option, duckdb 2.0 rejects duplicates Co-Authored-By: Claude Opus 5.5 * fix: dedupe quoted attach option names and drop comments from the list Co-Authored-By: Claude Opus 5.5 * fix: pin duckdb 2.0 from crates.io and revert more 2.0 defaults that change results Co-Authored-By: Claude Opus 5.5 * fix: fail duckdb jobs with a clear message on x86 cpus without avx2 Co-Authored-By: Claude Opus 5.5 * fix: strip sql comments from lake attach args before appending fork options Co-Authored-By: Claude Opus 5.5 * fix: scan E'' escape-string literals when stripping attach option comments Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/windmill-common/src/workspaces.rs | 97 ++++- .../windmill-duckdb-ffi-internal/Cargo.lock | 338 ++++++------------ .../windmill-duckdb-ffi-internal/Cargo.toml | 2 +- .../windmill-duckdb-ffi-internal/src/lib.rs | 50 +++ .../windmill-worker/src/duckdb_executor.rs | 176 +++++++-- backend/windmill-worker/src/duckdb_runner.rs | 17 + docs/duckdb-execution.md | 40 ++- frontend/src/lib/infer.svelte.ts | 7 +- 8 files changed, 464 insertions(+), 263 deletions(-) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 0a3bf0ac6f..248754dba8 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -2815,9 +2815,87 @@ pub fn fork_ducklake_ancestor_alias(lake_name: &str, ancestor_w_id: &str) -> Str ) } +/// The byte length of the string literal or quoted identifier starting at byte `i` of `sql` +/// ('…', "…", $tag$…$tag$, or E'…' with backslash escapes), if one starts there. An +/// unterminated one runs to the end. +pub fn sql_literal_len(sql: &str, i: usize) -> Option { + let rest = &sql[i..]; + let after_ident = sql[..i] + .chars() + .next_back() + .is_some_and(|c| c.is_alphanumeric() || c == '_'); + match rest.as_bytes()[0] { + b'E' | b'e' if !after_ident && rest[1..].starts_with('\'') => { + let bytes = rest.as_bytes(); + let mut j = 2; + while j < bytes.len() { + match bytes[j] { + b'\\' => j += 2, + b'\'' if bytes.get(j + 1) == Some(&b'\'') => j += 2, + b'\'' => return Some(j + 1), + _ => j += 1, + } + } + Some(rest.len()) + } + q @ (b'\'' | b'"') => Some(rest[1..].find(q as char).map_or(rest.len(), |e| e + 2)), + b'$' => rest[1..] + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .filter(|&n| rest[1 + n..].starts_with('$')) + .map(|n| { + let delim = &rest[..n + 2]; + rest[n + 2..] + .find(delim) + .map_or(rest.len(), |e| n + 2 + e + delim.len()) + }), + _ => None, + } +} + +/// Replaces each SQL comment outside string literals ([`sql_literal_len`]) with a space, +/// following DuckDB: `--` runs to the end of the line and `/* */` nests. An unterminated block +/// comment is kept as is, so the statement it ends up in fails to parse rather than silently +/// losing what follows. +pub fn strip_sql_comments(sql: &str) -> String { + let mut out = String::with_capacity(sql.len()); + let mut i = 0; + while i < sql.len() { + let rest = &sql[i..]; + if let Some(len) = sql_literal_len(sql, i) { + out.push_str(&rest[..len]); + i += len; + } else if rest.starts_with("--") { + i += rest.find('\n').unwrap_or(rest.len()); + out.push(' '); + } else if rest.starts_with("/*") { + let (mut depth, mut j) = (1usize, 2); + while depth > 0 && j < rest.len() { + if rest[j..].starts_with("/*") { + (depth, j) = (depth + 1, j + 2); + } else if rest[j..].starts_with("*/") { + (depth, j) = (depth - 1, j + 2); + } else { + j += rest[j..].chars().next().map_or(1, char::len_utf8); + } + } + if depth > 0 { + out.push_str(rest); + break; + } + out.push(' '); + i += j; + } else { + let c = rest.chars().next().unwrap(); + out.push(c); + i += c.len_utf8(); + } + } + out +} + /// Strip `METADATA_SCHEMA` / `DATA_PATH` / `OVERRIDE_DATA_PATH` tokens from ducklake ATTACH /// extra args. In a fork these options are injected by the fork resolution; a user- or -/// settings-supplied duplicate silently wins (DuckDB keeps the last occurrence) and would +/// settings-supplied duplicate silently wins (the worker keeps the last occurrence) and would /// escape the fork namespace back to the parent's, so they are removed rather than overridden. pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String { lazy_static::lazy_static! { @@ -2826,8 +2904,10 @@ pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String { ) .unwrap(); } + // Comments go first: the fork-owned options are appended after these args, and a trailing + // `-- …` would otherwise comment them out of the ATTACH. RESERVED - .replace_all(extra_args, "") + .replace_all(&strip_sql_comments(extra_args), "") .split(',') .map(str::trim) .filter(|s| !s.is_empty()) @@ -4280,6 +4360,19 @@ mod tests { strip_fork_reserved_attach_args("METADATA_SCHEMA 'wm_fork_evil'"), "" ); + // A trailing comment must not survive to comment out the fork options appended after. + assert_eq!( + strip_fork_reserved_attach_args("READ_ONLY false -- why, '/*'"), + "READ_ONLY false" + ); + assert_eq!( + strip_sql_comments("a /* x /* y */ x */ b '--' $q$/*$q$ -- c\nd /* open"), + "a b '--' $q$/*$q$ \nd /* open" + ); + assert_eq!( + strip_sql_comments(r"x E'a\'/* k */\'b' /* d */ e'c''/*'"), + r"x E'a\'/* k */\'b' e'c''/*'" + ); } #[test] diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index 74cd7b8a18..b5a36aeee8 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -51,15 +51,6 @@ dependencies = [ "libc", ] -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arrayvec" version = "0.7.8" @@ -68,9 +59,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14" dependencies = [ "arrow-arith", "arrow-array", @@ -86,9 +77,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720" dependencies = [ "arrow-array", "arrow-buffer", @@ -100,9 +91,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -111,6 +102,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -118,9 +110,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", @@ -130,9 +122,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -141,7 +133,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64", "chrono", "comfy-table", "half", @@ -151,10 +143,21 @@ dependencies = [ ] [[package]] -name = "arrow-data" -version = "58.4.0" +name = "arrow-cmp" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + +[[package]] +name = "arrow-data" +version = "60.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -165,12 +168,13 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -178,9 +182,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a" dependencies = [ "arrow-array", "arrow-buffer", @@ -191,22 +195,23 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" dependencies = [ "bitflags", ] [[package]] name = "arrow-select" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash 0.8.12", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -214,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1" dependencies = [ "arrow-array", "arrow-buffer", @@ -231,9 +236,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993" dependencies = [ "num-traits", ] @@ -244,12 +249,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64" version = "0.23.1" @@ -274,6 +273,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "borsh" version = "1.8.0" @@ -377,15 +385,20 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.4" +version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +checksum = "b96a46ad395a5e00676eddaca4a45ed78b29c08be5a878d35e4b071c625637dd" dependencies = [ - "crossterm", "unicode-segmentation", "unicode-width", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -412,6 +425,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -421,28 +443,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags", - "crossterm_winapi", - "parking_lot", - "rustix 0.38.44", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - [[package]] name = "crunchy" version = "0.2.4" @@ -450,25 +450,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] -name = "derive_arbitrary" -version = "1.4.2" +name = "crypto-common" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", ] [[package]] name = "duckdb" -version = "1.10506.0" +version = "1.20000.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3472b9152d879c5534353b20854e43525b2c78b2dbba156711e9939d30278b4" +checksum = "d807ca47d4fa206f102ff568046e018fb84518eb0a0b5e65a165a2eb7bd4a7c2" dependencies = [ "arrow", "cast", - "comfy-table", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -530,7 +538,6 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -670,6 +677,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -796,19 +812,19 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libduckdb-sys" -version = "1.10506.0" +version = "1.20000.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aac3459f2f7f0221d5be9114fa77ff5877040c8de456d3b986a680579fe202b" +checksum = "0140448c472968ee5cf5ba697435deebe558b366ef1e5c080d24d9696b600725" dependencies = [ "cc", "flate2", "pkg-config", "serde", "serde_json", + "sha2", "tar", "ureq", "vcpkg", - "zip", ] [[package]] @@ -817,27 +833,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "linux-raw-sys" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "log" version = "0.4.33" @@ -862,9 +863,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", @@ -904,29 +905,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1049,15 +1027,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "regex" version = "1.13.1" @@ -1156,19 +1125,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - [[package]] name = "rustix" version = "1.1.4" @@ -1178,7 +1134,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.12.1", + "linux-raw-sys", "windows-sys 0.61.2", ] @@ -1229,12 +1185,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "seahash" version = "4.1.0" @@ -1285,6 +1235,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1309,12 +1270,6 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - [[package]] name = "strum" version = "0.27.2" @@ -1446,6 +1401,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1476,7 +1437,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" dependencies = [ - "base64 0.23.1", + "base64", "log", "percent-encoding", "rustls", @@ -1492,7 +1453,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" dependencies = [ - "base64 0.23.1", + "base64", "http", "httparse", "log", @@ -1596,28 +1557,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windmill_duckdb_ffi_internal" version = "0.1.0" @@ -1698,15 +1637,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1811,7 +1741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.4", + "rustix", ] [[package]] @@ -1840,40 +1770,8 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -[[package]] -name = "zip" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" -dependencies = [ - "arbitrary", - "crc32fast", - "flate2", - "indexmap", - "memchr", - "zopfli", -] - -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index ba78e3522d..7a2bffc553 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -13,7 +13,7 @@ chrono = "0.4.41" # # Pinned exactly, so `cargo update` cannot move the engine (and the prebuilt extensions it # must match) under us. -duckdb = { version = "=1.10506.0", features = [ +duckdb = { version = "=1.20000.1", features = [ "bundled", "rust_decimal", ] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index 0de7d3bd79..3efbce038c 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -219,6 +219,22 @@ fn configure_duckdb_resource_limits( Ok(()) } +// DuckDB 2.0 defaults that would change what a script written for 1.x returns, reverted so it +// keeps running unchanged; a script can SET any of them back. Without `legacy_disable_null_type` +// an untyped `NULL` column has the NULL type, which duckdb-rs cannot read and panics on. The +// storage version keeps `.duckdb` files a script writes readable by 1.x. `lambda_syntax` is +// removed in DuckDB 2.1, which will make `x -> ...` lambdas an error again. +fn keep_duckdb_1x_semantics(conn: &duckdb::Connection) -> Result<(), String> { + conn.execute_batch( + "SET lambda_syntax='enable_single_arrow'; + SET error_on_division_by_zero=false; + SET legacy_disable_null_type=true; + SET regex_match_operator_semantics='full'; + SET storage_compatibility_version='v0.10.2';", + ) + .map_err(|e| format!("Error applying DuckDB 1.x compatibility settings: {e}")) +} + fn setup_duckdb_connection( conn: &duckdb::Connection, token: &str, @@ -227,6 +243,7 @@ fn setup_duckdb_connection( limits: &ResourceLimits, ) -> Result<(), String> { configure_duckdb_resource_limits(conn, limits)?; + keep_duckdb_1x_semantics(conn)?; let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token)); let (s3_endpoint_ssl, s3_endpoint) = base_internal_url @@ -240,8 +257,11 @@ fn setup_duckdb_connection( let s3_secret_key = sql_single_quote(s3_secret_key); let endpoint = sql_single_quote(&format!("{s3_endpoint}/api/w/{w_id}/s3_proxy")); + // `s3_uploader_max_filesize` is DuckDB 1.x's cap on one multipart upload; 2.0 lowered it to + // 80GB. It is an httpfs setting, so it can only be set once httpfs is loaded. conn.execute_batch(&format!( "INSTALL httpfs; LOAD httpfs; + SET s3_uploader_max_filesize='800GB'; INSTALL azure; LOAD azure; CREATE OR REPLACE SECRET s3_secret ( TYPE s3, @@ -1016,6 +1036,36 @@ mod temporal_json_tests { ); } + #[test] + fn scripts_written_for_duckdb_1x_keep_their_semantics() { + let conn = duckdb::Connection::open_in_memory().unwrap(); + keep_duckdb_1x_semantics(&conn).unwrap(); + let (lambda, int_div, modulo): (String, Option, Option) = conn + .query_row( + "SELECT list_transform([1, 2], x -> x + 1)::VARCHAR, 1 // 0, 1 % 0", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + assert_eq!(lambda, "[2, 3]"); + assert_eq!((int_div, modulo), (None, None)); + + // Read as a `Value`, the way results are rendered: an untyped NULL column panics there + // without `legacy_disable_null_type`. + let (untyped_null, nested_null, partial_match): ( + duckdb::types::Value, + duckdb::types::Value, + bool, + ) = conn + .query_row("SELECT NULL, {'a': NULL}, 'foobar' ~ 'foo'", [], |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?)) + }) + .unwrap(); + assert_eq!(untyped_null, duckdb::types::Value::Null); + assert!(matches!(nested_null, duckdb::types::Value::Struct(_))); + assert!(!partial_match, "`~` must keep matching the whole string"); + } + #[test] fn temporal_values_render_iso_through_real_query() { let conn = duckdb::Connection::open_in_memory().unwrap(); diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 0db954d85f..8c3bab10ed 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -10,7 +10,7 @@ use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ get_datatable_resource_from_db, get_ducklake_from_db_unchecked, - strip_fork_reserved_attach_args, DatatableAccess, DatatableConnectAs, + sql_literal_len, strip_fork_reserved_attach_args, strip_sql_comments, DatatableAccess, DatatableConnectAs, DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; @@ -1602,7 +1602,6 @@ pub async fn do_duckdb( ) .await?; - let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); let job_dir = job_dir.to_string(); @@ -2234,11 +2233,18 @@ async fn db_resource_to_attach_statements( ) .to_string(); - Ok(vec![ + let mut statements = vec![ get_attach_db_install_str(db_type)?.to_string(), format!("LOAD {};", db_type), - attach_str, - ]) + ]; + // DuckDB 2.0's MySQL scanner hands whole queries to the server by default, which turns + // binary-collated text into BLOB and moves grouping to MySQL's collation. Read when the + // ATTACH runs, so it has to precede it. + if db_type == "mysql" { + statements.push("SET mysql_enable_remote_pushdown=false;".to_string()); + } + statements.push(attach_str); + Ok(statements) } async fn transform_attach_ducklake( @@ -2267,7 +2273,7 @@ async fn transform_attach_ducklake( Connection::Sql(db) => get_ducklake_from_db_unchecked(name, w_id, db).await?, }; // In a fork, METADATA_SCHEMA / DATA_PATH / OVERRIDE_DATA_PATH are owned by the fork - // resolution (DuckDB silently keeps the last occurrence of a duplicated option, so a + // resolution (the last occurrence of a duplicated option is the one kept, so a // user-supplied one would escape the fork namespace back to the parent's). let user_extra_args = if ducklake.fork_defer.is_some() { strip_fork_reserved_attach_args(&user_extra_args) @@ -2308,37 +2314,18 @@ async fn transform_attach_ducklake( let data_path = ducklake.storage.path.replace('\'', "''"); let extra_args = if let Some(default_extra_args) = ducklake.extra_args { - format!("{},{}", extra_args, default_extra_args) + // A newline, so a trailing `--` comment in the script's args ends before the config's. + format!("{}\n,{}", extra_args, default_extra_args) } else { extra_args }; - // Ducklake 0.3 only requires DATA_PATH at creation and then stores it internally in the catalog - // But it will fail if DATA_PATH changes afterwards which is annoying for us - // So we always enable override - let extra_args = if extra_args.contains("OVERRIDE_DATA_PATH") { - extra_args - } else { - format!(", OVERRIDE_DATA_PATH TRUE{extra_args}") - }; - // Automatically migrate ducklake - let extra_args = if extra_args.contains("AUTOMATIC_MIGRATION") { - extra_args - } else { - format!(", AUTOMATIC_MIGRATION TRUE{extra_args}") - }; - // In a fork, re-emit the fork-owned DATA_PATH as the LAST option: DuckDB keeps the last - // occurrence of a duplicated option, so this wins over anything the arg stripping might - // not recognize (e.g. a dollar-quoted literal), regardless of literal syntax. The - // METADATA_SCHEMA injected by the fork resolution is already last within `extra_args`. - let extra_args = if ducklake.fork_defer.is_some() { - format!("{extra_args}, DATA_PATH 's3://{storage}/{data_path}'") - } else { - extra_args - }; - - let attach_str = format!( - "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});", + let options = ducklake_attach_options( + &format!("s3://{storage}/{data_path}"), + &extra_args, + ducklake.fork_defer.is_some(), ); + let attach_str = + format!("ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} ({options});"); let install_db_ext_str = get_attach_db_install_str(db_type)?; let mut statements = vec![ @@ -2359,6 +2346,86 @@ async fn transform_attach_ducklake( Ok(Some(statements)) } +/// The option list of a lake's ATTACH. `extra_args` are the script's then the lake config's, +/// each part prefixed by a comma; later options override earlier ones, so Windmill's defaults +/// come first and, in a fork, the fork-owned DATA_PATH last (it must win over anything the +/// reserved-arg stripping did not recognize, e.g. a dollar-quoted literal). +fn ducklake_attach_options(data_path: &str, extra_args: &str, fork: bool) -> String { + // OVERRIDE_DATA_PATH: DuckLake stores DATA_PATH at creation and refuses a different one + // afterwards. DUCKLAKE_VERSION '1.0': a catalog migrated to 1.1 cannot be downgraded and is + // unreadable by DuckDB 1.x engines, so one job on an upgraded worker would break every older + // worker and any rollback. Unpinned, AUTOMATIC_MIGRATION targets the newest format and new + // catalogs are created at it. + let mut options = format!( + "DATA_PATH '{data_path}', OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, \ + DUCKLAKE_VERSION '1.0'{extra_args}" + ); + if fork { + options.push_str(&format!("\n, DATA_PATH '{data_path}'")); + } + keep_last_attach_option(&options) +} + +/// Drops every occurrence of an ATTACH option but its last. DuckDB 1.x kept the last of +/// duplicated options; 2.0 rejects the statement ("Unexpected duplicate option"). +fn keep_last_attach_option(options: &str) -> String { + let parts = split_attach_options(options); + let key = |part: &str| -> String { + match part.strip_prefix('"') { + Some(quoted) => quoted.split('"').next().unwrap_or_default().to_string(), + None => part + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_') + .collect(), + } + .to_ascii_uppercase() + }; + let keys: Vec = parts.iter().map(|p| key(p)).collect(); + parts + .iter() + .enumerate() + .filter(|(i, _)| keys[i + 1..].iter().all(|k| k.is_empty() || *k != keys[*i])) + .map(|(_, p)| p.as_str()) + .collect::>() + .join(", ") +} + +/// Splits an ATTACH option list on the commas outside string literals ([`sql_literal_len`]) +/// and parentheses, dropping empty parts. SQL comments are dropped too: a `--` comment would +/// otherwise swallow whatever is joined after it on the same line, `);` included. +fn split_attach_options(options: &str) -> Vec { + let options = strip_sql_comments(options); + let options = options.as_str(); + let mut parts = vec![]; + let mut part = String::new(); + let mut depth = 0usize; + let mut i = 0; + while i < options.len() { + let rest = &options[i..]; + if let Some(len) = sql_literal_len(options, i) { + part.push_str(&rest[..len]); + i += len; + continue; + } + let c = rest.chars().next().unwrap(); + match c { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + ',' if depth == 0 => { + parts.push(std::mem::take(&mut part).trim().to_string()); + i += 1; + continue; + } + _ => {} + } + part.push(c); + i += c.len_utf8(); + } + parts.push(part.trim().to_string()); + parts.retain(|p| !p.is_empty()); + parts +} + // Double-quote a possibly schema-qualified table reference, each dotted segment // independently (local copy of sql_materialize's private helper). fn quote_qualified_table(name: &str) -> String { @@ -2420,8 +2487,8 @@ fn fork_defer_statements( .map(|s| format!(", METADATA_SCHEMA '{}'", s.replace('\'', "''"))) .unwrap_or_default(); // The ancestor config's own non-reserved args (e.g. `ENCRYPTED true`) — an - // option-dependent lake wouldn't attach without them. Emitted FIRST: DuckDB keeps the - // last occurrence of a duplicated option, so the fork-owned DATA_PATH / READ_ONLY / + // option-dependent lake wouldn't attach without them. Emitted FIRST: the last occurrence + // of a duplicated option is the one kept, so the fork-owned DATA_PATH / READ_ONLY / // METADATA_SCHEMA after them always win. let extra_args = a .extra_args @@ -2432,8 +2499,11 @@ fn fork_defer_statements( // / CREATE_IF_NOT_EXISTS: an ancestor lake that would need creating or migrating fails // loudly rather than being mutated from a fork. IF NOT EXISTS: several ATTACH blocks in // one script (e.g. the user's + the materialize synthetic one) emit the same ancestors. + let options = keep_last_attach_option(&format!( + "{extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema}" + )); stmts.push(format!( - "ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema});", + "ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({options});", a.alias )); } @@ -2906,6 +2976,40 @@ mod tests { ) } + #[test] + fn ducklake_attach_pins_catalog_format_and_never_repeats_an_option() { + assert_eq!( + ducklake_attach_options("s3://b/p", "", false), + "DATA_PATH 's3://b/p', OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, \ + DUCKLAKE_VERSION '1.0'" + ); + // The script's then the lake config's args override the defaults; in a fork the + // fork-owned DATA_PATH overrides everything. + assert_eq!( + ducklake_attach_options( + "s3://b/fork", + ", data_path 's3://b/parent', ducklake_version '1.1',METADATA_SCHEMA 'a,b'", + true + ), + "OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, ducklake_version '1.1', \ + METADATA_SCHEMA 'a,b', DATA_PATH 's3://b/fork'" + ); + assert_eq!( + keep_last_attach_option("X 'it''s, ok', ENCRYPTED $t$a, b$t$, X 1, f(a, b)"), + "ENCRYPTED $t$a, b$t$, X 1, f(a, b)" + ); + assert_eq!( + keep_last_attach_option( + "DATA_PATH 'a', \"data_path\" 'b', READ_ONLY false -- why, it is\n, /* x, */ E 1" + ), + "\"data_path\" 'b', READ_ONLY false, E 1" + ); + assert_eq!( + keep_last_attach_option("READ_ONLY false /* a /* b, */ c, */, E 1"), + "READ_ONLY false, E 1" + ); + } + #[test] fn test_fork_defer_statements_ancestor_extra_args() { // The ancestor config's own args (e.g. ENCRYPTED) must ride on its read-only attach — diff --git a/backend/windmill-worker/src/duckdb_runner.rs b/backend/windmill-worker/src/duckdb_runner.rs index 56b3c1f5cc..68e4912834 100644 --- a/backend/windmill-worker/src/duckdb_runner.rs +++ b/backend/windmill-worker/src/duckdb_runner.rs @@ -293,11 +293,28 @@ fn parse_duckdb_size(s: &str) -> Option { Some((num * mult) as u64) } +/// DuckDB's prebuilt linux_amd64 extensions target x86-64-v3, and every job loads httpfs, so +/// on a CPU without AVX2 the runner would die with SIGILL and an opaque crash message. +fn check_cpu_supported() -> Result<()> { + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] + if !std::arch::is_x86_feature_detected!("avx2") { + return Err(Error::ExecutionErr( + "DuckDB requires a CPU with AVX2 (x86-64-v3): its extensions crash on this worker's \ + CPU. On a VM, set the CPU type to `host` or `x86-64-v3` (Proxmox's default \ + `x86-64-v2-AES` hides AVX2; on VMware, raise the EVC mode to Haswell or later), or \ + run DuckDB jobs on a worker group with a newer CPU." + .to_string(), + )); + } + Ok(()) +} + pub(crate) async fn run( mode: RunnerMode, req: &RunnerRequest<'_>, job_dir: &str, ) -> Result { + check_cpu_supported()?; let spill_dir = format!("{}/spill", work_dir(job_dir)); tokio::fs::create_dir_all(&spill_dir).await?; let req = RunnerRequest { temp_directory: &spill_dir, ..*req }; diff --git a/docs/duckdb-execution.md b/docs/duckdb-execution.md index 70f9218ee4..42afe5fc5f 100644 --- a/docs/duckdb-execution.md +++ b/docs/duckdb-execution.md @@ -81,9 +81,43 @@ affects its own connection, and is no more than a Python job could do. ## The engine -The runner links duckdb-rs from crates.io, pinned with `=` so `cargo update` cannot move the -engine. The engine loads the **prebuilt** extensions from `extensions.duckdb.org`, compiled for the -exact release it reports, so an engine bump is a version change in `Cargo.toml` and nothing else. +The runner links duckdb-rs pinned exactly (`=`) so `cargo update` cannot move the engine. The engine loads the **prebuilt** extensions from `extensions.duckdb.org`, +compiled for the exact release it reports, so an engine bump is a version change in `Cargo.toml` +and nothing else. The reported version comes from the vendored sources' `pragma_version.cpp`: a +build that calls itself `vX.Y.Z-devN` downloads from a per-commit path that does not exist, and +every extension `INSTALL` 404s. Check `SELECT version()` after a bump. + +The engine is built from source with baseline flags, but the prebuilt 2.0 linux_amd64 extensions +are compiled for x86-64-v3: on a CPU without AVX2 (pre-Haswell, or a VM CPU model such as +Proxmox's `x86-64-v2-AES` or a low VMware EVC baseline) `LOAD httpfs` dies with SIGILL, and +every job loads httpfs. `duckdb_runner::run` therefore fails the job up front on such a CPU, with +a message naming the VM settings to change, rather than letting the runner crash. linux_arm64 +extensions run on Cortex-A53 and later. + +What a bump can break is user scripts and shared state, so a major one keeps the previous major's +behavior and leaves the new one opt-in: + +- `setup_duckdb_connection` reverts the defaults DuckDB 2.0 changed that would make a query that + worked on 1.x return something else: `x -> …` lambdas, `NULL` for integer `//` / `%` by zero, + untyped `NULL` columns typed INTEGER (the 2.0 NULL type makes duckdb-rs panic on read), + whole-string `~`, `.duckdb` files written at a version 1.x can open, and the 800GB S3 upload + cap. A script can `SET` any of them back. DuckDB 2.1 removes `lambda_syntax`, and the setting + has to go then. +- `ieee_floating_point_ops` stays at 2.0's default on purpose. Turning it off does not restore + 1.x: `/` by zero then returns `NULL` where 1.x returned infinity. Left on, the only change is + that `sqrt(-1)`, `ln(0)` and the like return NaN instead of erroring. +- MySQL resource attaches turn off `mysql_enable_remote_pushdown`, which is read when the + ATTACH runs. Left on, binary-collated text comes back as BLOB and grouping follows MySQL's + collation. +- What no setting restores: auto-generated names of unaliased expressions (`main.` prefixes + dropped), unnamed structs rendered as `element1`… instead of a single `""` key, JSON + auto-detection typing offset timestamps as TIMESTAMPTZ, `GROUP BY ALL` over an `UNNEST` + erroring, `X'..'` literals typed BLOB, and Parquet columns from constants written REQUIRED. +- `transform_attach_ducklake` attaches lakes with `DUCKLAKE_VERSION '1.0'` unless the lake's + extra args name one. Windmill also passes `AUTOMATIC_MIGRATION TRUE`, which unpinned migrates to + the newest catalog format. A migrated catalog cannot be downgraded and older engines cannot read + it, so a mixed-version worker fleet or a rollback would lose the lake. A lake opts into 1.1 + with `DUCKLAKE_VERSION '1.1'` in its extra args, once no worker runs DuckDB 1.x. Confinement belongs to the job isolation, not to engine settings. DuckDB spills through the same file system a query reads with, so `disabled_filesystems='LocalFileSystem'` also stops every query diff --git a/frontend/src/lib/infer.svelte.ts b/frontend/src/lib/infer.svelte.ts index cc34fc64fa..8e153d0780 100644 --- a/frontend/src/lib/infer.svelte.ts +++ b/frontend/src/lib/infer.svelte.ts @@ -70,9 +70,14 @@ type QueryEntry = [string, InferAssetsSqlQueryDetails] // DuckDB prepare replaces $N params with NULL. Some functions (read_parquet, read_csv, etc.) // reject NULL arguments, which is expected — the query will work at execution time with real args. +// DuckDB 1.x says "cannot take NULL"; 2.0 reports the NULL as an argument no overload accepts. function isNullParamSubstitutionError(error?: string): boolean { if (!error) return false - return error.includes('cannot take NULL') || error.includes('Could not choose a best candidate') + return ( + error.includes('cannot take NULL') || + error.includes('Could not choose a best candidate') || + error.includes('No function matches the given name and argument types') + ) } function mapPrepareResults(