mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-21 16:06:54 +00:00
feat: upgrade the bundled DuckDB engine to 2.0.0 LTS (#11924)
* feat: upgrade the bundled DuckDB engine to 2.0.0 LTS Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: never repeat a ducklake attach option, duckdb 2.0 rejects duplicates Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: dedupe quoted attach option names and drop comments from the list Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: pin duckdb 2.0 from crates.io and revert more 2.0 defaults that change results Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: fail duckdb jobs with a clear message on x86 cpus without avx2 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: strip sql comments from lake attach args before appending fork options Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: scan E'' escape-string literals when stripping attach option comments Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2aa6059e58
commit
3d0ca92ef2
@@ -2815,9 +2815,87 @@ pub fn fork_ducklake_ancestor_alias(lake_name: &str, ancestor_w_id: &str) -> Str
|
||||
)
|
||||
}
|
||||
|
||||
/// The byte length of the string literal or quoted identifier starting at byte `i` of `sql`
|
||||
/// ('…', "…", $tag$…$tag$, or E'…' with backslash escapes), if one starts there. An
|
||||
/// unterminated one runs to the end.
|
||||
pub fn sql_literal_len(sql: &str, i: usize) -> Option<usize> {
|
||||
let rest = &sql[i..];
|
||||
let after_ident = sql[..i]
|
||||
.chars()
|
||||
.next_back()
|
||||
.is_some_and(|c| c.is_alphanumeric() || c == '_');
|
||||
match rest.as_bytes()[0] {
|
||||
b'E' | b'e' if !after_ident && rest[1..].starts_with('\'') => {
|
||||
let bytes = rest.as_bytes();
|
||||
let mut j = 2;
|
||||
while j < bytes.len() {
|
||||
match bytes[j] {
|
||||
b'\\' => j += 2,
|
||||
b'\'' if bytes.get(j + 1) == Some(&b'\'') => j += 2,
|
||||
b'\'' => return Some(j + 1),
|
||||
_ => j += 1,
|
||||
}
|
||||
}
|
||||
Some(rest.len())
|
||||
}
|
||||
q @ (b'\'' | b'"') => Some(rest[1..].find(q as char).map_or(rest.len(), |e| e + 2)),
|
||||
b'$' => rest[1..]
|
||||
.find(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
|
||||
.filter(|&n| rest[1 + n..].starts_with('$'))
|
||||
.map(|n| {
|
||||
let delim = &rest[..n + 2];
|
||||
rest[n + 2..]
|
||||
.find(delim)
|
||||
.map_or(rest.len(), |e| n + 2 + e + delim.len())
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces each SQL comment outside string literals ([`sql_literal_len`]) with a space,
|
||||
/// following DuckDB: `--` runs to the end of the line and `/* */` nests. An unterminated block
|
||||
/// comment is kept as is, so the statement it ends up in fails to parse rather than silently
|
||||
/// losing what follows.
|
||||
pub fn strip_sql_comments(sql: &str) -> String {
|
||||
let mut out = String::with_capacity(sql.len());
|
||||
let mut i = 0;
|
||||
while i < sql.len() {
|
||||
let rest = &sql[i..];
|
||||
if let Some(len) = sql_literal_len(sql, i) {
|
||||
out.push_str(&rest[..len]);
|
||||
i += len;
|
||||
} else if rest.starts_with("--") {
|
||||
i += rest.find('\n').unwrap_or(rest.len());
|
||||
out.push(' ');
|
||||
} else if rest.starts_with("/*") {
|
||||
let (mut depth, mut j) = (1usize, 2);
|
||||
while depth > 0 && j < rest.len() {
|
||||
if rest[j..].starts_with("/*") {
|
||||
(depth, j) = (depth + 1, j + 2);
|
||||
} else if rest[j..].starts_with("*/") {
|
||||
(depth, j) = (depth - 1, j + 2);
|
||||
} else {
|
||||
j += rest[j..].chars().next().map_or(1, char::len_utf8);
|
||||
}
|
||||
}
|
||||
if depth > 0 {
|
||||
out.push_str(rest);
|
||||
break;
|
||||
}
|
||||
out.push(' ');
|
||||
i += j;
|
||||
} else {
|
||||
let c = rest.chars().next().unwrap();
|
||||
out.push(c);
|
||||
i += c.len_utf8();
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Strip `METADATA_SCHEMA` / `DATA_PATH` / `OVERRIDE_DATA_PATH` tokens from ducklake ATTACH
|
||||
/// extra args. In a fork these options are injected by the fork resolution; a user- or
|
||||
/// settings-supplied duplicate silently wins (DuckDB keeps the last occurrence) and would
|
||||
/// settings-supplied duplicate silently wins (the worker keeps the last occurrence) and would
|
||||
/// escape the fork namespace back to the parent's, so they are removed rather than overridden.
|
||||
pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String {
|
||||
lazy_static::lazy_static! {
|
||||
@@ -2826,8 +2904,10 @@ pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String {
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
// Comments go first: the fork-owned options are appended after these args, and a trailing
|
||||
// `-- …` would otherwise comment them out of the ATTACH.
|
||||
RESERVED
|
||||
.replace_all(extra_args, "")
|
||||
.replace_all(&strip_sql_comments(extra_args), "")
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
@@ -4280,6 +4360,19 @@ mod tests {
|
||||
strip_fork_reserved_attach_args("METADATA_SCHEMA 'wm_fork_evil'"),
|
||||
""
|
||||
);
|
||||
// A trailing comment must not survive to comment out the fork options appended after.
|
||||
assert_eq!(
|
||||
strip_fork_reserved_attach_args("READ_ONLY false -- why, '/*'"),
|
||||
"READ_ONLY false"
|
||||
);
|
||||
assert_eq!(
|
||||
strip_sql_comments("a /* x /* y */ x */ b '--' $q$/*$q$ -- c\nd /* open"),
|
||||
"a b '--' $q$/*$q$ \nd /* open"
|
||||
);
|
||||
assert_eq!(
|
||||
strip_sql_comments(r"x E'a\'/* k */\'b' /* d */ e'c''/*'"),
|
||||
r"x E'a\'/* k */\'b' e'c''/*'"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+118
-220
@@ -51,15 +51,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.8"
|
||||
@@ -68,9 +59,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
|
||||
checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -86,9 +77,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-arith"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
|
||||
checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -100,9 +91,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-array"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
|
||||
checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"arrow-buffer",
|
||||
@@ -111,6 +102,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"half",
|
||||
"hashbrown 0.17.1",
|
||||
"libc",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
@@ -118,9 +110,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-buffer"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
|
||||
checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"half",
|
||||
@@ -130,9 +122,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-cast"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
|
||||
checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -141,7 +133,7 @@ dependencies = [
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"atoi",
|
||||
"base64 0.22.1",
|
||||
"base64",
|
||||
"chrono",
|
||||
"comfy-table",
|
||||
"half",
|
||||
@@ -151,10 +143,21 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "58.4.0"
|
||||
name = "arrow-cmp"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
|
||||
checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f"
|
||||
dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
@@ -165,12 +168,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ord"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
|
||||
checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cmp",
|
||||
"arrow-data",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
@@ -178,9 +182,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-row"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
|
||||
checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -191,22 +195,23 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-schema"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
|
||||
checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrow-select"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
|
||||
checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
"arrow-cmp",
|
||||
"arrow-data",
|
||||
"arrow-schema",
|
||||
"num-traits",
|
||||
@@ -214,9 +219,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-string"
|
||||
version = "58.4.0"
|
||||
version = "60.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
|
||||
checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -231,9 +236,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "atoi"
|
||||
version = "2.0.0"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
|
||||
checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
@@ -244,12 +249,6 @@ version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
@@ -274,6 +273,15 @@ dependencies = [
|
||||
"wyz",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
|
||||
dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "1.8.0"
|
||||
@@ -377,15 +385,20 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "comfy-table"
|
||||
version = "7.1.4"
|
||||
version = "8.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a"
|
||||
checksum = "b96a46ad395a5e00676eddaca4a45ed78b29c08be5a878d35e4b071c625637dd"
|
||||
dependencies = [
|
||||
"crossterm",
|
||||
"unicode-segmentation",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "const-random"
|
||||
version = "0.1.18"
|
||||
@@ -412,6 +425,15 @@ version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
@@ -421,28 +443,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crossterm_winapi",
|
||||
"parking_lot",
|
||||
"rustix 0.38.44",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossterm_winapi"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
@@ -450,25 +450,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.4.2"
|
||||
name = "crypto-common"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||
checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"const-oid",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "duckdb"
|
||||
version = "1.10506.0"
|
||||
version = "1.20000.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3472b9152d879c5534353b20854e43525b2c78b2dbba156711e9939d30278b4"
|
||||
checksum = "d807ca47d4fa206f102ff568046e018fb84518eb0a0b5e65a165a2eb7bd4a7c2"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"cast",
|
||||
"comfy-table",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
@@ -530,7 +538,6 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
"zlib-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -670,6 +677,15 @@ version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.65"
|
||||
@@ -796,19 +812,19 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libduckdb-sys"
|
||||
version = "1.10506.0"
|
||||
version = "1.20000.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0aac3459f2f7f0221d5be9114fa77ff5877040c8de456d3b986a680579fe202b"
|
||||
checksum = "0140448c472968ee5cf5ba697435deebe558b366ef1e5c080d24d9696b600725"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"flate2",
|
||||
"pkg-config",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tar",
|
||||
"ureq",
|
||||
"vcpkg",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -817,27 +833,12 @@ version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.33"
|
||||
@@ -862,9 +863,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.8"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
|
||||
checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0"
|
||||
dependencies = [
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
@@ -904,29 +905,6 @@ version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
@@ -1049,15 +1027,6 @@ dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
@@ -1156,19 +1125,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
@@ -1178,7 +1134,7 @@ dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -1229,12 +1185,6 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "seahash"
|
||||
version = "4.1.0"
|
||||
@@ -1285,6 +1235,17 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
@@ -1309,12 +1270,6 @@ version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.27.2"
|
||||
@@ -1446,6 +1401,12 @@ dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
@@ -1476,7 +1437,7 @@ version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"rustls",
|
||||
@@ -1492,7 +1453,7 @@ version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"base64",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
@@ -1596,28 +1557,6 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windmill_duckdb_ffi_internal"
|
||||
version = "0.1.0"
|
||||
@@ -1698,15 +1637,6 @@ dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
@@ -1811,7 +1741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix 1.1.4",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1840,40 +1770,8 @@ version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
@@ -13,7 +13,7 @@ chrono = "0.4.41"
|
||||
#
|
||||
# Pinned exactly, so `cargo update` cannot move the engine (and the prebuilt extensions it
|
||||
# must match) under us.
|
||||
duckdb = { version = "=1.10506.0", features = [
|
||||
duckdb = { version = "=1.20000.1", features = [
|
||||
"bundled",
|
||||
"rust_decimal",
|
||||
] }
|
||||
|
||||
@@ -219,6 +219,22 @@ fn configure_duckdb_resource_limits(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// DuckDB 2.0 defaults that would change what a script written for 1.x returns, reverted so it
|
||||
// keeps running unchanged; a script can SET any of them back. Without `legacy_disable_null_type`
|
||||
// an untyped `NULL` column has the NULL type, which duckdb-rs cannot read and panics on. The
|
||||
// storage version keeps `.duckdb` files a script writes readable by 1.x. `lambda_syntax` is
|
||||
// removed in DuckDB 2.1, which will make `x -> ...` lambdas an error again.
|
||||
fn keep_duckdb_1x_semantics(conn: &duckdb::Connection) -> Result<(), String> {
|
||||
conn.execute_batch(
|
||||
"SET lambda_syntax='enable_single_arrow';
|
||||
SET error_on_division_by_zero=false;
|
||||
SET legacy_disable_null_type=true;
|
||||
SET regex_match_operator_semantics='full';
|
||||
SET storage_compatibility_version='v0.10.2';",
|
||||
)
|
||||
.map_err(|e| format!("Error applying DuckDB 1.x compatibility settings: {e}"))
|
||||
}
|
||||
|
||||
fn setup_duckdb_connection(
|
||||
conn: &duckdb::Connection,
|
||||
token: &str,
|
||||
@@ -227,6 +243,7 @@ fn setup_duckdb_connection(
|
||||
limits: &ResourceLimits,
|
||||
) -> Result<(), String> {
|
||||
configure_duckdb_resource_limits(conn, limits)?;
|
||||
keep_duckdb_1x_semantics(conn)?;
|
||||
|
||||
let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token));
|
||||
let (s3_endpoint_ssl, s3_endpoint) = base_internal_url
|
||||
@@ -240,8 +257,11 @@ fn setup_duckdb_connection(
|
||||
let s3_secret_key = sql_single_quote(s3_secret_key);
|
||||
let endpoint = sql_single_quote(&format!("{s3_endpoint}/api/w/{w_id}/s3_proxy"));
|
||||
|
||||
// `s3_uploader_max_filesize` is DuckDB 1.x's cap on one multipart upload; 2.0 lowered it to
|
||||
// 80GB. It is an httpfs setting, so it can only be set once httpfs is loaded.
|
||||
conn.execute_batch(&format!(
|
||||
"INSTALL httpfs; LOAD httpfs;
|
||||
SET s3_uploader_max_filesize='800GB';
|
||||
INSTALL azure; LOAD azure;
|
||||
CREATE OR REPLACE SECRET s3_secret (
|
||||
TYPE s3,
|
||||
@@ -1016,6 +1036,36 @@ mod temporal_json_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scripts_written_for_duckdb_1x_keep_their_semantics() {
|
||||
let conn = duckdb::Connection::open_in_memory().unwrap();
|
||||
keep_duckdb_1x_semantics(&conn).unwrap();
|
||||
let (lambda, int_div, modulo): (String, Option<i64>, Option<i64>) = conn
|
||||
.query_row(
|
||||
"SELECT list_transform([1, 2], x -> x + 1)::VARCHAR, 1 // 0, 1 % 0",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(lambda, "[2, 3]");
|
||||
assert_eq!((int_div, modulo), (None, None));
|
||||
|
||||
// Read as a `Value`, the way results are rendered: an untyped NULL column panics there
|
||||
// without `legacy_disable_null_type`.
|
||||
let (untyped_null, nested_null, partial_match): (
|
||||
duckdb::types::Value,
|
||||
duckdb::types::Value,
|
||||
bool,
|
||||
) = conn
|
||||
.query_row("SELECT NULL, {'a': NULL}, 'foobar' ~ 'foo'", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(untyped_null, duckdb::types::Value::Null);
|
||||
assert!(matches!(nested_null, duckdb::types::Value::Struct(_)));
|
||||
assert!(!partial_match, "`~` must keep matching the whole string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn temporal_values_render_iso_through_real_query() {
|
||||
let conn = duckdb::Connection::open_in_memory().unwrap();
|
||||
|
||||
@@ -10,7 +10,7 @@ use windmill_common::utils::sanitize_string_from_password;
|
||||
use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy};
|
||||
use windmill_common::workspaces::{
|
||||
get_datatable_resource_from_db, get_ducklake_from_db_unchecked,
|
||||
strip_fork_reserved_attach_args, DatatableAccess, DatatableConnectAs,
|
||||
sql_literal_len, strip_fork_reserved_attach_args, strip_sql_comments, DatatableAccess, DatatableConnectAs,
|
||||
DucklakeCatalogResourceType,
|
||||
};
|
||||
use windmill_common::PgDatabase;
|
||||
@@ -1602,7 +1602,6 @@ pub async fn do_duckdb(
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
let base_internal_url = client.base_internal_url.clone();
|
||||
let w_id = job.workspace_id.clone();
|
||||
let job_dir = job_dir.to_string();
|
||||
@@ -2234,11 +2233,18 @@ async fn db_resource_to_attach_statements(
|
||||
)
|
||||
.to_string();
|
||||
|
||||
Ok(vec![
|
||||
let mut statements = vec![
|
||||
get_attach_db_install_str(db_type)?.to_string(),
|
||||
format!("LOAD {};", db_type),
|
||||
attach_str,
|
||||
])
|
||||
];
|
||||
// DuckDB 2.0's MySQL scanner hands whole queries to the server by default, which turns
|
||||
// binary-collated text into BLOB and moves grouping to MySQL's collation. Read when the
|
||||
// ATTACH runs, so it has to precede it.
|
||||
if db_type == "mysql" {
|
||||
statements.push("SET mysql_enable_remote_pushdown=false;".to_string());
|
||||
}
|
||||
statements.push(attach_str);
|
||||
Ok(statements)
|
||||
}
|
||||
|
||||
async fn transform_attach_ducklake(
|
||||
@@ -2267,7 +2273,7 @@ async fn transform_attach_ducklake(
|
||||
Connection::Sql(db) => get_ducklake_from_db_unchecked(name, w_id, db).await?,
|
||||
};
|
||||
// In a fork, METADATA_SCHEMA / DATA_PATH / OVERRIDE_DATA_PATH are owned by the fork
|
||||
// resolution (DuckDB silently keeps the last occurrence of a duplicated option, so a
|
||||
// resolution (the last occurrence of a duplicated option is the one kept, so a
|
||||
// user-supplied one would escape the fork namespace back to the parent's).
|
||||
let user_extra_args = if ducklake.fork_defer.is_some() {
|
||||
strip_fork_reserved_attach_args(&user_extra_args)
|
||||
@@ -2308,37 +2314,18 @@ async fn transform_attach_ducklake(
|
||||
let data_path = ducklake.storage.path.replace('\'', "''");
|
||||
|
||||
let extra_args = if let Some(default_extra_args) = ducklake.extra_args {
|
||||
format!("{},{}", extra_args, default_extra_args)
|
||||
// A newline, so a trailing `--` comment in the script's args ends before the config's.
|
||||
format!("{}\n,{}", extra_args, default_extra_args)
|
||||
} else {
|
||||
extra_args
|
||||
};
|
||||
// Ducklake 0.3 only requires DATA_PATH at creation and then stores it internally in the catalog
|
||||
// But it will fail if DATA_PATH changes afterwards which is annoying for us
|
||||
// So we always enable override
|
||||
let extra_args = if extra_args.contains("OVERRIDE_DATA_PATH") {
|
||||
extra_args
|
||||
} else {
|
||||
format!(", OVERRIDE_DATA_PATH TRUE{extra_args}")
|
||||
};
|
||||
// Automatically migrate ducklake
|
||||
let extra_args = if extra_args.contains("AUTOMATIC_MIGRATION") {
|
||||
extra_args
|
||||
} else {
|
||||
format!(", AUTOMATIC_MIGRATION TRUE{extra_args}")
|
||||
};
|
||||
// In a fork, re-emit the fork-owned DATA_PATH as the LAST option: DuckDB keeps the last
|
||||
// occurrence of a duplicated option, so this wins over anything the arg stripping might
|
||||
// not recognize (e.g. a dollar-quoted literal), regardless of literal syntax. The
|
||||
// METADATA_SCHEMA injected by the fork resolution is already last within `extra_args`.
|
||||
let extra_args = if ducklake.fork_defer.is_some() {
|
||||
format!("{extra_args}, DATA_PATH 's3://{storage}/{data_path}'")
|
||||
} else {
|
||||
extra_args
|
||||
};
|
||||
|
||||
let attach_str = format!(
|
||||
"ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});",
|
||||
let options = ducklake_attach_options(
|
||||
&format!("s3://{storage}/{data_path}"),
|
||||
&extra_args,
|
||||
ducklake.fork_defer.is_some(),
|
||||
);
|
||||
let attach_str =
|
||||
format!("ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} ({options});");
|
||||
|
||||
let install_db_ext_str = get_attach_db_install_str(db_type)?;
|
||||
let mut statements = vec![
|
||||
@@ -2359,6 +2346,86 @@ async fn transform_attach_ducklake(
|
||||
Ok(Some(statements))
|
||||
}
|
||||
|
||||
/// The option list of a lake's ATTACH. `extra_args` are the script's then the lake config's,
|
||||
/// each part prefixed by a comma; later options override earlier ones, so Windmill's defaults
|
||||
/// come first and, in a fork, the fork-owned DATA_PATH last (it must win over anything the
|
||||
/// reserved-arg stripping did not recognize, e.g. a dollar-quoted literal).
|
||||
fn ducklake_attach_options(data_path: &str, extra_args: &str, fork: bool) -> String {
|
||||
// OVERRIDE_DATA_PATH: DuckLake stores DATA_PATH at creation and refuses a different one
|
||||
// afterwards. DUCKLAKE_VERSION '1.0': a catalog migrated to 1.1 cannot be downgraded and is
|
||||
// unreadable by DuckDB 1.x engines, so one job on an upgraded worker would break every older
|
||||
// worker and any rollback. Unpinned, AUTOMATIC_MIGRATION targets the newest format and new
|
||||
// catalogs are created at it.
|
||||
let mut options = format!(
|
||||
"DATA_PATH '{data_path}', OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, \
|
||||
DUCKLAKE_VERSION '1.0'{extra_args}"
|
||||
);
|
||||
if fork {
|
||||
options.push_str(&format!("\n, DATA_PATH '{data_path}'"));
|
||||
}
|
||||
keep_last_attach_option(&options)
|
||||
}
|
||||
|
||||
/// Drops every occurrence of an ATTACH option but its last. DuckDB 1.x kept the last of
|
||||
/// duplicated options; 2.0 rejects the statement ("Unexpected duplicate option").
|
||||
fn keep_last_attach_option(options: &str) -> String {
|
||||
let parts = split_attach_options(options);
|
||||
let key = |part: &str| -> String {
|
||||
match part.strip_prefix('"') {
|
||||
Some(quoted) => quoted.split('"').next().unwrap_or_default().to_string(),
|
||||
None => part
|
||||
.chars()
|
||||
.take_while(|c| c.is_ascii_alphanumeric() || *c == '_')
|
||||
.collect(),
|
||||
}
|
||||
.to_ascii_uppercase()
|
||||
};
|
||||
let keys: Vec<String> = parts.iter().map(|p| key(p)).collect();
|
||||
parts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| keys[i + 1..].iter().all(|k| k.is_empty() || *k != keys[*i]))
|
||||
.map(|(_, p)| p.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ")
|
||||
}
|
||||
|
||||
/// Splits an ATTACH option list on the commas outside string literals ([`sql_literal_len`])
|
||||
/// and parentheses, dropping empty parts. SQL comments are dropped too: a `--` comment would
|
||||
/// otherwise swallow whatever is joined after it on the same line, `);` included.
|
||||
fn split_attach_options(options: &str) -> Vec<String> {
|
||||
let options = strip_sql_comments(options);
|
||||
let options = options.as_str();
|
||||
let mut parts = vec![];
|
||||
let mut part = String::new();
|
||||
let mut depth = 0usize;
|
||||
let mut i = 0;
|
||||
while i < options.len() {
|
||||
let rest = &options[i..];
|
||||
if let Some(len) = sql_literal_len(options, i) {
|
||||
part.push_str(&rest[..len]);
|
||||
i += len;
|
||||
continue;
|
||||
}
|
||||
let c = rest.chars().next().unwrap();
|
||||
match c {
|
||||
'(' => depth += 1,
|
||||
')' => depth = depth.saturating_sub(1),
|
||||
',' if depth == 0 => {
|
||||
parts.push(std::mem::take(&mut part).trim().to_string());
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
part.push(c);
|
||||
i += c.len_utf8();
|
||||
}
|
||||
parts.push(part.trim().to_string());
|
||||
parts.retain(|p| !p.is_empty());
|
||||
parts
|
||||
}
|
||||
|
||||
// Double-quote a possibly schema-qualified table reference, each dotted segment
|
||||
// independently (local copy of sql_materialize's private helper).
|
||||
fn quote_qualified_table(name: &str) -> String {
|
||||
@@ -2420,8 +2487,8 @@ fn fork_defer_statements(
|
||||
.map(|s| format!(", METADATA_SCHEMA '{}'", s.replace('\'', "''")))
|
||||
.unwrap_or_default();
|
||||
// The ancestor config's own non-reserved args (e.g. `ENCRYPTED true`) — an
|
||||
// option-dependent lake wouldn't attach without them. Emitted FIRST: DuckDB keeps the
|
||||
// last occurrence of a duplicated option, so the fork-owned DATA_PATH / READ_ONLY /
|
||||
// option-dependent lake wouldn't attach without them. Emitted FIRST: the last occurrence
|
||||
// of a duplicated option is the one kept, so the fork-owned DATA_PATH / READ_ONLY /
|
||||
// METADATA_SCHEMA after them always win.
|
||||
let extra_args = a
|
||||
.extra_args
|
||||
@@ -2432,8 +2499,11 @@ fn fork_defer_statements(
|
||||
// / CREATE_IF_NOT_EXISTS: an ancestor lake that would need creating or migrating fails
|
||||
// loudly rather than being mutated from a fork. IF NOT EXISTS: several ATTACH blocks in
|
||||
// one script (e.g. the user's + the materialize synthetic one) emit the same ancestors.
|
||||
let options = keep_last_attach_option(&format!(
|
||||
"{extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema}"
|
||||
));
|
||||
stmts.push(format!(
|
||||
"ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema});",
|
||||
"ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({options});",
|
||||
a.alias
|
||||
));
|
||||
}
|
||||
@@ -2906,6 +2976,40 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ducklake_attach_pins_catalog_format_and_never_repeats_an_option() {
|
||||
assert_eq!(
|
||||
ducklake_attach_options("s3://b/p", "", false),
|
||||
"DATA_PATH 's3://b/p', OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, \
|
||||
DUCKLAKE_VERSION '1.0'"
|
||||
);
|
||||
// The script's then the lake config's args override the defaults; in a fork the
|
||||
// fork-owned DATA_PATH overrides everything.
|
||||
assert_eq!(
|
||||
ducklake_attach_options(
|
||||
"s3://b/fork",
|
||||
", data_path 's3://b/parent', ducklake_version '1.1',METADATA_SCHEMA 'a,b'",
|
||||
true
|
||||
),
|
||||
"OVERRIDE_DATA_PATH TRUE, AUTOMATIC_MIGRATION TRUE, ducklake_version '1.1', \
|
||||
METADATA_SCHEMA 'a,b', DATA_PATH 's3://b/fork'"
|
||||
);
|
||||
assert_eq!(
|
||||
keep_last_attach_option("X 'it''s, ok', ENCRYPTED $t$a, b$t$, X 1, f(a, b)"),
|
||||
"ENCRYPTED $t$a, b$t$, X 1, f(a, b)"
|
||||
);
|
||||
assert_eq!(
|
||||
keep_last_attach_option(
|
||||
"DATA_PATH 'a', \"data_path\" 'b', READ_ONLY false -- why, it is\n, /* x, */ E 1"
|
||||
),
|
||||
"\"data_path\" 'b', READ_ONLY false, E 1"
|
||||
);
|
||||
assert_eq!(
|
||||
keep_last_attach_option("READ_ONLY false /* a /* b, */ c, */, E 1"),
|
||||
"READ_ONLY false, E 1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fork_defer_statements_ancestor_extra_args() {
|
||||
// The ancestor config's own args (e.g. ENCRYPTED) must ride on its read-only attach —
|
||||
|
||||
@@ -293,11 +293,28 @@ fn parse_duckdb_size(s: &str) -> Option<u64> {
|
||||
Some((num * mult) as u64)
|
||||
}
|
||||
|
||||
/// DuckDB's prebuilt linux_amd64 extensions target x86-64-v3, and every job loads httpfs, so
|
||||
/// on a CPU without AVX2 the runner would die with SIGILL and an opaque crash message.
|
||||
fn check_cpu_supported() -> Result<()> {
|
||||
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
|
||||
if !std::arch::is_x86_feature_detected!("avx2") {
|
||||
return Err(Error::ExecutionErr(
|
||||
"DuckDB requires a CPU with AVX2 (x86-64-v3): its extensions crash on this worker's \
|
||||
CPU. On a VM, set the CPU type to `host` or `x86-64-v3` (Proxmox's default \
|
||||
`x86-64-v2-AES` hides AVX2; on VMware, raise the EVC mode to Haswell or later), or \
|
||||
run DuckDB jobs on a worker group with a newer CPU."
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn run(
|
||||
mode: RunnerMode,
|
||||
req: &RunnerRequest<'_>,
|
||||
job_dir: &str,
|
||||
) -> Result<RunnerOutput> {
|
||||
check_cpu_supported()?;
|
||||
let spill_dir = format!("{}/spill", work_dir(job_dir));
|
||||
tokio::fs::create_dir_all(&spill_dir).await?;
|
||||
let req = RunnerRequest { temp_directory: &spill_dir, ..*req };
|
||||
|
||||
@@ -81,9 +81,43 @@ affects its own connection, and is no more than a Python job could do.
|
||||
|
||||
## The engine
|
||||
|
||||
The runner links duckdb-rs from crates.io, pinned with `=` so `cargo update` cannot move the
|
||||
engine. The engine loads the **prebuilt** extensions from `extensions.duckdb.org`, compiled for the
|
||||
exact release it reports, so an engine bump is a version change in `Cargo.toml` and nothing else.
|
||||
The runner links duckdb-rs pinned exactly (`=`) so `cargo update` cannot move the engine. The engine loads the **prebuilt** extensions from `extensions.duckdb.org`,
|
||||
compiled for the exact release it reports, so an engine bump is a version change in `Cargo.toml`
|
||||
and nothing else. The reported version comes from the vendored sources' `pragma_version.cpp`: a
|
||||
build that calls itself `vX.Y.Z-devN` downloads from a per-commit path that does not exist, and
|
||||
every extension `INSTALL` 404s. Check `SELECT version()` after a bump.
|
||||
|
||||
The engine is built from source with baseline flags, but the prebuilt 2.0 linux_amd64 extensions
|
||||
are compiled for x86-64-v3: on a CPU without AVX2 (pre-Haswell, or a VM CPU model such as
|
||||
Proxmox's `x86-64-v2-AES` or a low VMware EVC baseline) `LOAD httpfs` dies with SIGILL, and
|
||||
every job loads httpfs. `duckdb_runner::run` therefore fails the job up front on such a CPU, with
|
||||
a message naming the VM settings to change, rather than letting the runner crash. linux_arm64
|
||||
extensions run on Cortex-A53 and later.
|
||||
|
||||
What a bump can break is user scripts and shared state, so a major one keeps the previous major's
|
||||
behavior and leaves the new one opt-in:
|
||||
|
||||
- `setup_duckdb_connection` reverts the defaults DuckDB 2.0 changed that would make a query that
|
||||
worked on 1.x return something else: `x -> …` lambdas, `NULL` for integer `//` / `%` by zero,
|
||||
untyped `NULL` columns typed INTEGER (the 2.0 NULL type makes duckdb-rs panic on read),
|
||||
whole-string `~`, `.duckdb` files written at a version 1.x can open, and the 800GB S3 upload
|
||||
cap. A script can `SET` any of them back. DuckDB 2.1 removes `lambda_syntax`, and the setting
|
||||
has to go then.
|
||||
- `ieee_floating_point_ops` stays at 2.0's default on purpose. Turning it off does not restore
|
||||
1.x: `/` by zero then returns `NULL` where 1.x returned infinity. Left on, the only change is
|
||||
that `sqrt(-1)`, `ln(0)` and the like return NaN instead of erroring.
|
||||
- MySQL resource attaches turn off `mysql_enable_remote_pushdown`, which is read when the
|
||||
ATTACH runs. Left on, binary-collated text comes back as BLOB and grouping follows MySQL's
|
||||
collation.
|
||||
- What no setting restores: auto-generated names of unaliased expressions (`main.` prefixes
|
||||
dropped), unnamed structs rendered as `element1`… instead of a single `""` key, JSON
|
||||
auto-detection typing offset timestamps as TIMESTAMPTZ, `GROUP BY ALL` over an `UNNEST`
|
||||
erroring, `X'..'` literals typed BLOB, and Parquet columns from constants written REQUIRED.
|
||||
- `transform_attach_ducklake` attaches lakes with `DUCKLAKE_VERSION '1.0'` unless the lake's
|
||||
extra args name one. Windmill also passes `AUTOMATIC_MIGRATION TRUE`, which unpinned migrates to
|
||||
the newest catalog format. A migrated catalog cannot be downgraded and older engines cannot read
|
||||
it, so a mixed-version worker fleet or a rollback would lose the lake. A lake opts into 1.1
|
||||
with `DUCKLAKE_VERSION '1.1'` in its extra args, once no worker runs DuckDB 1.x.
|
||||
|
||||
Confinement belongs to the job isolation, not to engine settings. DuckDB spills through the same
|
||||
file system a query reads with, so `disabled_filesystems='LocalFileSystem'` also stops every query
|
||||
|
||||
@@ -70,9 +70,14 @@ type QueryEntry = [string, InferAssetsSqlQueryDetails]
|
||||
|
||||
// DuckDB prepare replaces $N params with NULL. Some functions (read_parquet, read_csv, etc.)
|
||||
// reject NULL arguments, which is expected — the query will work at execution time with real args.
|
||||
// DuckDB 1.x says "cannot take NULL"; 2.0 reports the NULL as an argument no overload accepts.
|
||||
function isNullParamSubstitutionError(error?: string): boolean {
|
||||
if (!error) return false
|
||||
return error.includes('cannot take NULL') || error.includes('Could not choose a best candidate')
|
||||
return (
|
||||
error.includes('cannot take NULL') ||
|
||||
error.includes('Could not choose a best candidate') ||
|
||||
error.includes('No function matches the given name and argument types')
|
||||
)
|
||||
}
|
||||
|
||||
function mapPrepareResults(
|
||||
|
||||
Reference in New Issue
Block a user