starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-05 00:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: duckdb sql lang support (#5761)

Browse Source
This commit is contained in:
Diego Imbert
2025-05-27 15:52:57 +02:00
committed by GitHub
parent c049d4230f
commit d5e9213fd5
41 changed files with 1310 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.gitignore
+2 -1
View File
@@ -5,4 +5,5 @@ oauth2.json
tracing.folded
heaptrack*
index/
windmill-api/openapi-*.*
windmill-api/openapi-*.*
.duckdb/*
backend/Cargo.lock
Generated
+288 -88
View File
@@ -286,25 +286,57 @@ dependencies = [
"serde",
]
[[package]]
name = "arrow"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796"
dependencies = [
"arrow-arith 54.3.1",
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-cast 54.3.1",
"arrow-data 54.3.1",
"arrow-ord 54.3.1",
"arrow-row 54.3.1",
"arrow-schema 54.3.1",
"arrow-select 54.3.1",
"arrow-string 54.3.1",
]
[[package]]
name = "arrow"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1bb018b6960c87fd9d025009820406f74e83281185a8bdcb44880d2aa5c9a87"
dependencies = [
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-arith 55.1.0",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-cast 55.1.0",
"arrow-csv",
"arrow-data",
"arrow-data 55.1.0",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-row",
"arrow-schema",
"arrow-select",
"arrow-string",
"arrow-ord 55.1.0",
"arrow-row 55.1.0",
"arrow-schema 55.1.0",
"arrow-select 55.1.0",
"arrow-string 55.1.0",
]
[[package]]
name = "arrow-arith"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e"
dependencies = [
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"chrono",
"num",
]
[[package]]
@@ -313,14 +345,30 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44de76b51473aa888ecd6ad93ceb262fb8d40d1f1154a4df2f069b3590aa7575"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"chrono",
"num",
]
[[package]]
name = "arrow-array"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985"
dependencies = [
"ahash 0.8.12",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"chrono",
"half",
"hashbrown 0.15.3",
"num",
]
[[package]]
name = "arrow-array"
version = "55.1.0"
@@ -328,9 +376,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29ed77e22744475a9a53d00026cf8e166fe73cf42d89c4c4ae63607ee1cfcc3f"
dependencies = [
"ahash 0.8.12",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"chrono",
"chrono-tz",
"half",
@@ -338,6 +386,17 @@ dependencies = [
"num",
]
[[package]]
name = "arrow-buffer"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c"
dependencies = [
"bytes",
"half",
"num",
]
[[package]]
name = "arrow-buffer"
version = "55.1.0"
@@ -349,17 +408,38 @@ dependencies = [
"num",
]
[[package]]
name = "arrow-cast"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9"
dependencies = [
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"arrow-select 54.3.1",
"atoi",
"base64 0.22.1",
"chrono",
"comfy-table",
"half",
"lexical-core",
"num",
"ryu",
]
[[package]]
name = "arrow-cast"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f39e1d774ece9292697fcbe06b5584401b26bd34be1bec25c33edae65c2420ff"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"arrow-select 55.1.0",
"atoi",
"base64 0.22.1",
"chrono",
@@ -376,9 +456,9 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9055c972a07bf12c2a827debfd34f88d3b93da1941d36e1d9fee85eebe38a12a"
dependencies = [
"arrow-array",
"arrow-cast",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-cast 55.1.0",
"arrow-schema 55.1.0",
"chrono",
"csv",
"csv-core",
@@ -386,14 +466,26 @@ dependencies = [
"regex",
]
[[package]]
name = "arrow-data"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429"
dependencies = [
"arrow-buffer 54.3.1",
"arrow-schema 54.3.1",
"half",
"num",
]
[[package]]
name = "arrow-data"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf75ac27a08c7f48b88e5c923f267e980f27070147ab74615ad85b5c5f90473d"
dependencies = [
"arrow-buffer",
"arrow-schema",
"arrow-buffer 55.1.0",
"arrow-schema 55.1.0",
"half",
"num",
]
@@ -404,10 +496,10 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a222f0d93772bd058d1268f4c28ea421a603d66f7979479048c429292fac7b2e"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"flatbuffers",
"lz4_flex",
]
@@ -418,11 +510,11 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9085342bbca0f75e8cb70513c0807cc7351f1fbf5cb98192a67d5e3044acb033"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-cast 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"chrono",
"half",
"indexmap 2.9.0",
@@ -434,17 +526,43 @@ dependencies = [
"simdutf8",
]
[[package]]
name = "arrow-ord"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21"
dependencies = [
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"arrow-select 54.3.1",
]
[[package]]
name = "arrow-ord"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab2f1065a5cad7b9efa9e22ce5747ce826aa3855766755d4904535123ef431e7"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"arrow-select 55.1.0",
]
[[package]]
name = "arrow-row"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68"
dependencies = [
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"half",
]
[[package]]
@@ -453,19 +571,42 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3703a0e3e92d23c3f756df73d2dc9476873f873a76ae63ef9d3de17fda83b2d8"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"half",
]
[[package]]
name = "arrow-schema"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
dependencies = [
"bitflags 2.9.1",
]
[[package]]
name = "arrow-schema"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b"
[[package]]
name = "arrow-select"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619"
dependencies = [
"ahash 0.8.12",
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"num",
]
[[package]]
name = "arrow-select"
version = "55.1.0"
@@ -473,24 +614,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24b7b85575702b23b85272b01bc1c25a01c9b9852305e5d0078c79ba25d995d4"
dependencies = [
"ahash 0.8.12",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"num",
]
[[package]]
name = "arrow-string"
version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03"
dependencies = [
"arrow-array 54.3.1",
"arrow-buffer 54.3.1",
"arrow-data 54.3.1",
"arrow-schema 54.3.1",
"arrow-select 54.3.1",
"memchr",
"num",
"regex",
"regex-syntax 0.8.5",
]
[[package]]
name = "arrow-string"
version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9260fddf1cdf2799ace2b4c2fc0356a9789fa7551e0953e35435536fecefebbd"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-data 55.1.0",
"arrow-schema 55.1.0",
"arrow-select 55.1.0",
"memchr",
"num",
"regex",
@@ -1838,6 +1996,12 @@ dependencies = [
"syn 2.0.101",
]
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cbc"
version = "0.1.2"
@@ -2599,9 +2763,9 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080"
dependencies = [
"arrow",
"arrow 55.1.0",
"arrow-ipc",
"arrow-schema",
"arrow-schema 55.1.0",
"async-trait",
"bytes",
"bzip2",
@@ -2654,7 +2818,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"dashmap 6.1.0",
"datafusion-common",
@@ -2680,7 +2844,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"datafusion-catalog",
"datafusion-common",
@@ -2704,7 +2868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow 55.1.0",
"arrow-ipc",
"base64 0.22.1",
"half",
@@ -2738,7 +2902,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-compression",
"async-trait",
"bytes",
@@ -2774,7 +2938,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"bytes",
"datafusion-catalog",
@@ -2799,7 +2963,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"bytes",
"datafusion-catalog",
@@ -2824,7 +2988,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"bytes",
"datafusion-catalog",
@@ -2861,7 +3025,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84"
dependencies = [
"arrow",
"arrow 55.1.0",
"dashmap 6.1.0",
"datafusion-common",
"datafusion-expr",
@@ -2880,7 +3044,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964"
dependencies = [
"arrow",
"arrow 55.1.0",
"chrono",
"datafusion-common",
"datafusion-doc",
@@ -2901,7 +3065,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839"
dependencies = [
"arrow",
"arrow 55.1.0",
"datafusion-common",
"indexmap 2.9.0",
"itertools 0.14.0",
@@ -2914,8 +3078,8 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e"
dependencies = [
"arrow",
"arrow-buffer",
"arrow 55.1.0",
"arrow-buffer 55.1.0",
"base64 0.22.1",
"blake2",
"blake3",
@@ -2944,7 +3108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow 55.1.0",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
@@ -2965,7 +3129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow 55.1.0",
"datafusion-common",
"datafusion-expr-common",
"datafusion-physical-expr-common",
@@ -2977,8 +3141,8 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537"
dependencies = [
"arrow",
"arrow-ord",
"arrow 55.1.0",
"arrow-ord 55.1.0",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
@@ -2998,7 +3162,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"datafusion-catalog",
"datafusion-common",
@@ -3052,7 +3216,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515"
dependencies = [
"arrow",
"arrow 55.1.0",
"chrono",
"datafusion-common",
"datafusion-expr",
@@ -3072,7 +3236,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow 55.1.0",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
@@ -3094,7 +3258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow 55.1.0",
"datafusion-common",
"datafusion-expr-common",
"hashbrown 0.14.5",
@@ -3107,7 +3271,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2"
dependencies = [
"arrow",
"arrow 55.1.0",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
@@ -3127,9 +3291,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55"
dependencies = [
"ahash 0.8.12",
"arrow",
"arrow-ord",
"arrow-schema",
"arrow 55.1.0",
"arrow-ord 55.1.0",
"arrow-schema 55.1.0",
"async-trait",
"chrono",
"datafusion-common",
@@ -3156,7 +3320,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f"
dependencies = [
"arrow",
"arrow 55.1.0",
"async-trait",
"dashmap 6.1.0",
"datafusion-common",
@@ -3180,7 +3344,7 @@ version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607"
dependencies = [
"arrow",
"arrow 55.1.0",
"bigdecimal",
"datafusion-common",
"datafusion-expr",
@@ -4640,6 +4804,25 @@ dependencies = [
"zeroize",
]
[[package]]
name = "duckdb"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49ac283b6621e3becf8014d1efa655522794075834c72f744573debef9c9f6c8"
dependencies = [
"arrow 54.3.1",
"cast",
"fallible-iterator 0.3.0",
"fallible-streaming-iterator",
"hashlink 0.9.1",
"libduckdb-sys",
"memchr",
"num-integer",
"rust_decimal",
"smallvec",
"strum 0.25.0",
]
[[package]]
name = "dunce"
version = "1.0.5"
@@ -7149,6 +7332,22 @@ version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]]
name = "libduckdb-sys"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12cac9d03484c43fefac8b2066a253c9b0b3b0cd02cbe02a9ea2312f7e382618"
dependencies = [
"autocfg",
"cc",
"flate2",
"pkg-config",
"serde",
"serde_json",
"tar",
"vcpkg",
]
[[package]]
name = "libffi"
version = "3.2.0"
@@ -8950,13 +9149,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be7b2d778f6b841d37083ebdf32e33a524acde1266b5884a8ca29bf00dfa1231"
dependencies = [
"ahash 0.8.12",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-array 55.1.0",
"arrow-buffer 55.1.0",
"arrow-cast 55.1.0",
"arrow-data 55.1.0",
"arrow-ipc",
"arrow-schema",
"arrow-select",
"arrow-schema 55.1.0",
"arrow-select 55.1.0",
"base64 0.22.1",
"brotli 8.0.1",
"bytes",
@@ -15056,6 +15255,7 @@ dependencies = [
"deno_web",
"deno_webidl",
"dotenv",
"duckdb",
"dyn-iter",
"flume",
"futures",
backend/Cargo.toml
+3 -1
View File
@@ -87,13 +87,14 @@ python = ["windmill-worker/python", "windmill-api/python"]
rust = ["windmill-worker/rust"]
mysql = ["windmill-worker/mysql"]
oracledb = ["windmill-worker/oracledb"]
duckdb = ["windmill-worker/duckdb"]
mssql = ["windmill-worker/mssql"]
bigquery = ["windmill-worker/bigquery"]
php = ["windmill-worker/php"]
csharp = ["windmill-worker/csharp"]
nu = ["windmill-worker/nu"]
java = ["windmill-worker/java"]
all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql", "bigquery", "csharp", "nu", "php", "java"]
[patch.crates-io]
@@ -238,6 +239,7 @@ json-pointer = "^0"
itertools = "^0"
regex = "^1"
semver = "^1"
duckdb = { version = "1.2.2", features = ["bundled"] }
v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix
deno_fetch = "0.214.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
6899b8151329218a1df59964dac57e0e004ae25a
32039f675060b5996951708368bdefe14278d5cd
backend/migrations/20250515084520_duckdb_support.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20250515084520_duckdb_support.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'duckdb';
UPDATE config set config = jsonb_set(config, '{worker_tags}', config->'worker_tags' || '["duckdb"]'::jsonb) where name = 'worker__default' and config @> '{"worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "php", "rust", "ansible", "csharp", "nu", "java"]}'::jsonb AND NOT config->'worker_tags' @> '"duckdb"'::jsonb;
backend/parsers/windmill-parser-sql/src/lib.rs
+74
View File
@@ -83,6 +83,21 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
}
}
pub fn parse_duckdb_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_duckdb_file(&code)?;
if let Some(args) = parsed {
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
}
pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_snowflake_file(&code)?;
if let Some(x) = parsed {
@@ -212,6 +227,9 @@ lazy_static::lazy_static! {
// -- @name (type) = default
static ref RE_ARG_BIGQUERY: Regex = Regex::new(r#"(?m)^-- @(\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
// -- $name (type) = default
static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
@@ -577,6 +595,35 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
Ok(Some(args))
}
fn parse_duckdb_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let mut args: Vec<Arg> = vec![];
for cap in RE_ARG_DUCKDB.captures_iter(code) {
let name = cap.get(1).map(|x| x.as_str().to_string()).unwrap();
let typ = cap
.get(2)
.map(|x| x.as_str().to_string().to_lowercase())
.unwrap();
let default = cap.get(3).map(|x| x.as_str().to_string());
let has_default = default.is_some();
let parsed_typ = parse_duckdb_typ(typ.as_str());
let parsed_default = default.and_then(|x| parsed_default(&parsed_typ, x));
args.push(Arg {
name,
typ: parsed_typ,
default: parsed_default,
otyp: Some(typ),
has_default,
oidx: None,
});
}
args.append(&mut parse_sql_sanitized_interpolation(code));
Ok(Some(args))
}
fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let mut args: Vec<Arg> = vec![];
@@ -729,6 +776,33 @@ pub fn parse_bigquery_typ(typ: &str) -> Typ {
}
}
pub fn parse_duckdb_typ(typ: &str) -> Typ {
if typ.ends_with("[]") {
let base_typ = parse_duckdb_typ(typ.strip_suffix("[]").unwrap());
Typ::List(Box::new(base_typ))
} else {
match typ {
"varchar" | "char" | "bpchar" | "text" | "string" => Typ::Str(None),
"blob" | "bytea" | "binary" | "varbinary" | "bitstring" => Typ::Bytes,
"boolean" | "bool" | "bit" | "logical" => Typ::Bool,
"bigint" | "int8" | "long" | "integer" | "int4" | "int" | "smallint" | "int2"
| "short" | "tinyint" | "int1" | "signed" | "ubigint" | "uhugeint" | "uinteger"
| "usmallint" | "utinyint" => Typ::Int,
"decimal" | "numeric" | "double" | "float8" | "float" | "float4" | "real" => Typ::Float,
"date"
| "time"
| "timestamp with time zone"
| "timestamptz"
| "timestamp"
| "datetime" => Typ::Datetime,
"uuid" | "json" => Typ::Str(None),
"interval" | "hugeint" => Typ::Str(None),
"s3object" => Typ::Resource("S3Object".to_string()),
_ => Typ::Str(None),
}
}
}
pub fn parse_snowflake_typ(typ: &str) -> Typ {
match typ {
"varchar" => Typ::Str(None),
backend/parsers/windmill-parser-wasm/src/lib.rs
+6
View File
@@ -96,6 +96,12 @@ pub fn parse_oracledb(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_oracledb_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_duckdb(code: &str) -> String {
wrap_sig(windmill_parser_sql::parse_duckdb_sig(code))
}
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_bigquery(code: &str) -> String {
backend/windmill-api/openapi.yaml
+2 -1
View File
@@ -14242,7 +14242,8 @@ components:
ansible,
csharp,
nu,
java
java,
duckdb
# for related places search: ADD_NEW_LANG
]
backend/windmill-api/src/workspaces_export.rs
+1
View File
@@ -360,6 +360,7 @@ pub(crate) async fn tarball_workspace(
ScriptLang::Bigquery => "bq.sql",
ScriptLang::Snowflake => "sf.sql",
ScriptLang::Mssql => "ms.sql",
ScriptLang::DuckDb => "duckdb.sql",
ScriptLang::Graphql => "gql",
ScriptLang::Nativets => "fetch.ts",
ScriptLang::Bun | ScriptLang::Bunnative => {
backend/windmill-common/src/client.rs
+45 -1
View File
@@ -2,7 +2,11 @@ use anyhow::Context;
use reqwest::{Body, Response};
use serde::de::DeserializeOwned;
use crate::utils::HTTP_CLIENT;
use crate::{
error::{self, to_anyhow},
s3_helpers::{DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse},
utils::HTTP_CLIENT,
};
#[derive(Clone)]
pub struct AuthedClient {
@@ -197,4 +201,44 @@ impl AuthedClient {
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
}
pub async fn get_duckdb_connection_settings(
&self,
s3: &DuckdbConnectionSettingsQueryV2,
) -> error::Result<DuckdbConnectionSettingsResponse> {
let url = format!(
"{}/api/w/{}/job_helpers/v2/duckdb_connection_settings",
self.base_internal_url, &self.workspace
);
let response = self
.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
.post(url)
.header(
reqwest::header::CONTENT_TYPE,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::ACCEPT,
reqwest::header::HeaderValue::from_static("application/json"),
)
.header(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", self.token))
.map_err(|e| error::Error::BadConfig(e.to_string()))?,
)
.body(serde_json::to_string(&s3).map_err(to_anyhow)?)
.send()
.await
.context(format!("Sent get_duckdb_connection_settings request",))
.map_err(error::Error::from)?;
match response.status().as_u16() {
200u16 => Ok(response
.json::<DuckdbConnectionSettingsResponse>()
.await
.context("decoding duckdb_connection_settings response as json")?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default()))?,
}
}
}
backend/windmill-common/src/s3_helpers.rs
+17
View File
@@ -891,3 +891,20 @@ pub async fn convert_json_line_stream<E: Into<anyhow::Error>>(
Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
}
#[derive(Deserialize, Serialize)]
pub struct DuckdbConnectionSettingsResponse {
pub connection_settings_str: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub azure_container_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub s3_bucket: Option<String>,
}
#[derive(Deserialize, Serialize)]
pub struct DuckdbConnectionSettingsQueryV2 {
#[serde(skip_serializing_if = "Option::is_none")]
pub s3_resource_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub storage: Option<String>,
}
backend/windmill-common/src/schema.rs
+1 -1
View File
@@ -349,7 +349,7 @@ pub fn should_validate_schema(code: &str, lang: &ScriptLang) -> bool {
let comment = match lang {
Nativets | Bun | Bunnative | Deno | Php | CSharp | Java => "//",
Python3 | Go | Bash | Powershell | Graphql | Ansible | Nu => "#",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB => "--",
Postgresql | Mysql | Bigquery | Snowflake | Mssql | OracleDB | DuckDb => "--",
Rust => "//!",
// for related places search: ADD_NEW_LANG
};
backend/windmill-common/src/scripts.rs
+3 -2
View File
@@ -46,13 +46,13 @@ pub enum ScriptLang {
Graphql,
Mssql,
OracleDB,
DuckDb,
Php,
Rust,
Ansible,
CSharp,
Nu,
Java,
// for related places search: ADD_NEW_LANG
Java, // for related places search: ADD_NEW_LANG
}
impl ScriptLang {
@@ -73,6 +73,7 @@ impl ScriptLang {
ScriptLang::Mssql => "mssql",
ScriptLang::Graphql => "graphql",
ScriptLang::OracleDB => "oracledb",
ScriptLang::DuckDb => "duckdb",
ScriptLang::Php => "php",
ScriptLang::Rust => "rust",
ScriptLang::Ansible => "ansible",
backend/windmill-common/src/worker.rs
+1
View File
@@ -79,6 +79,7 @@ lazy_static::lazy_static! {
"csharp".to_string(),
"nu".to_string(),
"java".to_string(),
"duckdb".to_string(),
// for related places search: ADD_NEW_LANG
"dependency".to_string(),
"flow".to_string(),
backend/windmill-worker/Cargo.toml
+2
View File
@@ -31,6 +31,7 @@ csharp = ["dep:windmill-parser-csharp"]
rust = ["dep:windmill-parser-rust"]
nu = ["dep:windmill-parser-nu"]
java = ["dep:windmill-parser-java"]
duckdb = []
[dependencies]
windmill-queue.workspace = true
@@ -92,6 +93,7 @@ deno_permissions = { workspace = true, optional = true }
deno_io = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
async-stream.workspace = true
duckdb.workspace = true
postgres-native-tls.workspace = true
native-tls.workspace = true
backend/windmill-worker/src/duckdb_executor.rs
+664
View File
@@ -0,0 +1,664 @@
use std::collections::HashMap;
use std::env;
use duckdb::types::TimeUnit;
use duckdb::{params_from_iter, Row};
use rust_decimal::prelude::FromPrimitive;
use rust_decimal::Decimal;
use serde_json::value::RawValue;
use serde_json::{json, Value};
use tokio::fs::remove_file;
use tokio::task;
use uuid::Uuid;
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::s3_helpers::{
DuckdbConnectionSettingsQueryV2, DuckdbConnectionSettingsResponse, S3Object,
};
use windmill_common::worker::{to_raw_value, Connection};
use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks};
use windmill_queue::{CanceledBy, MiniPulledJob};
use crate::common::{build_args_values, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
#[cfg(feature = "mysql")]
use crate::mysql_executor::MysqlDatabase;
use crate::pg_executor::PgDatabase;
use crate::sanitized_sql_params::sanitize_and_interpolate_unsafe_sql_args;
use windmill_common::client::AuthedClient;
fn do_duckdb_inner(
conn: &duckdb::Connection,
query: &str,
job_args: &HashMap<String, duckdb::types::Value>,
skip_collect: bool,
column_order: &mut Option<Vec<String>>,
) -> Result<Box<RawValue>> {
let mut rows_vec = vec![];
let (query, job_args) = interpolate_named_args(query, &job_args);
let mut stmt = conn
.prepare(&query)
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
let mut rows = stmt
.query(params_from_iter(job_args))
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
if skip_collect {
return Ok(to_raw_value(&json!([])));
}
// Statement needs to be stepped at least once or stmt.column_names() will panic
let mut column_names = None;
loop {
let row = rows.next();
match row {
Ok(Some(row)) => {
// Set column names if not already set
let stmt = row.as_ref();
let column_names = match column_names.as_ref() {
Some(column_names) => column_names,
None => {
column_names = Some(stmt.column_names());
column_names.as_ref().unwrap()
}
};
let row = row_to_value(row, &column_names.as_slice())
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
rows_vec.push(row);
}
Ok(None) => break,
Err(e) => {
return Err(Error::ExecutionErr(e.to_string()));
}
}
}
if let (Some(column_order), Some(column_names)) = (column_order.as_mut(), column_names) {
*column_order = column_names.clone();
}
return Ok(to_raw_value(&rows_vec));
}
pub async fn do_duckdb(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order_ref: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>> {
let result_f = async {
let sig = parse_duckdb_sig(query)?.args;
let mut job_args = build_args_values(job, client, conn).await?;
let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args)?;
// Prevent interpolate_named_args from detecting argument identifiers in the signature for
// the first query block
let query = trunc_sig(query);
let (_query_with_transformed_s3_uris, mut used_storages) =
transform_s3_uris(query, client).await?;
let query = _query_with_transformed_s3_uris.as_deref().unwrap_or(query);
let job_args = {
let mut m: HashMap<String, duckdb::types::Value> = HashMap::new();
for sig_arg in sig.into_iter() {
let json_value = job_args
.remove(&sig_arg.name)
.or_else(|| sig_arg.default)
.unwrap_or_else(|| json!(null));
if matches!(&sig_arg.otyp.as_ref().map(String::as_str), Some("s3object")) {
let s3_obj = serde_json::from_value::<S3Object>(json_value).map_err(|e| {
Error::ExecutionErr(format!("Failed to deserialize S3Object: {}", e))
})?;
let duckdb_conn_settings: windmill_common::s3_helpers::DuckdbConnectionSettingsResponse = client
.get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
s3_resource_path: None,
storage: s3_obj.storage.clone(),
})
.await?;
let uri = match (
&duckdb_conn_settings.s3_bucket,
&duckdb_conn_settings.azure_container_path,
) {
(Some(s3_bucket), None) => format!("s3://{}/{}", s3_bucket, &s3_obj.s3),
(None, Some(az_container)) => format!("{}/{}", az_container, &s3_obj.s3),
_ => {
return Err(Error::ExecutionErr(
"S3Object must have either s3_bucket or azure_container_path"
.to_string(),
));
}
};
m.insert(sig_arg.name, duckdb::types::Value::Text(uri));
used_storages.insert(s3_obj.storage, duckdb_conn_settings);
} else {
let duckdb_value = json_value_to_duckdb_value(
&json_value,
sig_arg
.otyp
.clone()
.unwrap_or_else(|| "text".to_string())
.as_str(),
client,
)?;
m.insert(sig_arg.name, duckdb_value);
}
}
m
};
let query_block_list = parse_sql_blocks(query);
// Replace windmill resource ATTACH statements with the real instructions
let query_block_list = {
let mut v = vec![];
for query_block in query_block_list.iter() {
match parse_attach_db_resource(query_block) {
Some(parsed) => v.extend(
transform_attach_db_resource_query(&parsed, &job.id, client).await?,
),
None => v.push(query_block.to_string()),
};
}
v
};
// duckdb::Connection is not Send so we do it in a single blocking task
let (result, column_order) = task::spawn_blocking(move || {
let conn = duckdb::Connection::open_in_memory()
.map_err(|e| Error::ConnectingToDatabase(e.to_string()))?;
for (_, DuckdbConnectionSettingsResponse { connection_settings_str, .. }) in
used_storages.into_iter()
{
conn.execute_batch(&connection_settings_str)
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
}
let mut result: Option<Box<RawValue>> = None;
let mut column_order = None;
for (query_block_index, query_block) in query_block_list.iter().enumerate() {
result = Some(
do_duckdb_inner(
&conn,
query_block.as_str(),
&job_args,
query_block_index != query_block_list.len() - 1,
&mut column_order,
)
.map_err(|e| Error::ExecutionErr(e.to_string()))?,
);
}
let result = result.unwrap_or_else(|| to_raw_value(&json!([])));
Ok::<_, Error>((result, column_order))
})
.await
.map_err(to_anyhow)??;
*column_order_ref = column_order;
// BigQuery cleanup
let bq_credentials_path = make_bq_credentials_path(&job.id);
env::remove_var("GOOGLE_APPLICATION_CREDENTIALS");
if matches!(tokio::fs::try_exists(&bq_credentials_path).await, Ok(true)) {
remove_file(&bq_credentials_path).await.map_err(to_anyhow)?;
}
Ok(result)
};
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
conn,
mem_peak,
canceled_by,
result_f,
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
Ok(result)
}
fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result<Box<RawValue>> {
let mut obj = serde_json::Map::new();
for (i, key) in column_names.iter().enumerate() {
let value: duckdb::types::Value =
row.get(i).map_err(|e| Error::ExecutionErr(e.to_string()))?;
let json_value = match value {
duckdb::types::Value::Null => serde_json::Value::Null,
duckdb::types::Value::Boolean(b) => serde_json::Value::Bool(b),
duckdb::types::Value::TinyInt(i) => serde_json::Value::Number(i.into()),
duckdb::types::Value::SmallInt(i) => serde_json::Value::Number(i.into()),
duckdb::types::Value::Int(i) => serde_json::Value::Number(i.into()),
duckdb::types::Value::BigInt(i) => serde_json::Value::Number(i.into()),
duckdb::types::Value::HugeInt(i) => serde_json::Value::String(i.to_string()),
duckdb::types::Value::UTinyInt(u) => serde_json::Value::Number(u.into()),
duckdb::types::Value::USmallInt(u) => serde_json::Value::Number(u.into()),
duckdb::types::Value::UInt(u) => serde_json::Value::Number(u.into()),
duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()),
duckdb::types::Value::Float(f) => serde_json::Value::Number(
serde_json::Number::from_f64(f as f64)
.ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
),
duckdb::types::Value::Double(f) => serde_json::Value::Number(
serde_json::Number::from_f64(f)
.ok_or_else(|| Error::ExecutionErr("Could not convert to f64".to_string()))?,
),
duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()),
duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()),
duckdb::types::Value::Text(s) => serde_json::Value::String(s),
duckdb::types::Value::Blob(b) => serde_json::Value::Array(
b.into_iter()
.map(|byte| serde_json::Value::Number(byte.into()))
.collect(),
),
duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()),
duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()),
duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({
"months": months,
"days": days,
"nanos": nanos
}),
duckdb::types::Value::List(values) => serde_json::Value::Array(
values
.into_iter()
.map(|v| serde_json::Value::String(format!("{:?}", v)))
.collect(),
),
duckdb::types::Value::Enum(e) => serde_json::Value::String(e),
duckdb::types::Value::Struct(fields) => serde_json::Value::Object(
fields
.iter()
.map(|(k, v)| (k.clone(), serde_json::Value::String(format!("{:?}", v))))
.collect(),
),
duckdb::types::Value::Array(values) => serde_json::Value::Array(
values
.into_iter()
.map(|v| serde_json::Value::String(format!("{:?}", v)))
.collect(),
),
duckdb::types::Value::Map(map) => serde_json::Value::Object(
map.iter()
.map(|(k, v)| {
(
format!("{:?}", k),
serde_json::Value::String(format!("{:?}", v)),
)
})
.collect(),
),
duckdb::types::Value::Union(value) => {
serde_json::Value::String(format!("{:?}", *value))
}
};
obj.insert(key.clone(), json_value);
}
serde_json::value::to_raw_value(&obj).map_err(|e| e.into())
}
fn json_value_to_duckdb_value(
json_value: &serde_json::Value,
arg_type: &str,
client: &AuthedClient,
) -> Result<duckdb::types::Value> {
let arg_type = arg_type.to_lowercase();
let duckdb_value = match json_value {
serde_json::Value::Null => duckdb::types::Value::Null,
serde_json::Value::Bool(b) => duckdb::types::Value::Boolean(*b),
serde_json::Value::String(s)
if matches!(
arg_type.as_str(),
"timestamp" | "timestamptz" | "timestamp with time zone" | "datetime"
) =>
{
string_to_duckdb_timestamp(&s)?
}
serde_json::Value::String(s) if arg_type.as_str() == "date" => string_to_duckdb_date(&s)?,
serde_json::Value::String(s) if arg_type.as_str() == "time" => string_to_duckdb_time(&s)?,
serde_json::Value::String(s) => duckdb::types::Value::Text(s.clone()),
serde_json::Value::Number(n) if n.is_i64() => {
let v = n.as_i64().unwrap();
match arg_type.as_str() {
"tinyint" | "int1" => duckdb::types::Value::TinyInt(v as i8),
"smallint" | "int2" | "short" => duckdb::types::Value::SmallInt(v as i16),
"integer" | "int4" | "int" | "signed" => duckdb::types::Value::Int(v as i32),
"bigint" | "int8" | "long" => duckdb::types::Value::BigInt(v),
"hugeint" => duckdb::types::Value::HugeInt(v as i128),
"float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
"double" | "float8" => duckdb::types::Value::Double(v as f64),
_ => duckdb::types::Value::BigInt(v), // default fallback
}
}
serde_json::Value::Number(n) if n.is_u64() => {
let v = n.as_u64().unwrap();
match arg_type.as_str() {
"utinyint" => duckdb::types::Value::UTinyInt(v as u8),
"usmallint" => duckdb::types::Value::USmallInt(v as u16),
"uinteger" => duckdb::types::Value::UInt(v as u32),
"ubigint" | "uhugeint" => duckdb::types::Value::UBigInt(v),
_ => duckdb::types::Value::UBigInt(v), // default fallback
}
}
serde_json::Value::Number(n) if n.is_f64() => {
let v = n.as_f64().unwrap();
match arg_type.as_str() {
"float" | "float4" | "real" => duckdb::types::Value::Float(v as f32),
"double" | "float8" => duckdb::types::Value::Double(v),
"decimal" | "numeric" => {
duckdb::types::Value::Decimal(Decimal::from_f64(v).ok_or_else(|| {
Error::ExecutionErr("Could not convert f64 to Decimal".to_string())
})?)
}
_ => duckdb::types::Value::Double(v), // default fallback
}
}
serde_json::Value::Array(arr) => duckdb::types::Value::Array(
arr.iter()
.map(|val| json_value_to_duckdb_value(val, arg_type.as_str(), client))
.collect::<Result<Vec<_>>>()?,
),
serde_json::Value::Object(map) => duckdb::types::Value::Struct(
map.iter()
.map(|(k, v)| {
Ok::<_, Error>((
k.clone(),
json_value_to_duckdb_value(v, arg_type.as_str(), client)?,
))
})
.collect::<Result<Vec<_>>>()?
.into(),
),
value @ _ => {
return Err(Error::ExecutionErr(format!(
"Unsupported type in query: {:?} and signature {arg_type:?}",
value
)))
}
};
Ok(duckdb_value)
}
fn string_to_duckdb_timestamp(s: &str) -> Result<duckdb::types::Value> {
let ts = chrono::DateTime::parse_from_rfc3339(s)
.map_err(|e: chrono::ParseError| Error::ExecutionErr(e.to_string()))?;
Ok(duckdb::types::Value::Timestamp(
TimeUnit::Millisecond,
ts.timestamp_millis(),
))
}
fn string_to_duckdb_date(s: &str) -> Result<duckdb::types::Value> {
use chrono::Datelike;
let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap();
Ok(duckdb::types::Value::Date32(date.num_days_from_ce()))
}
fn string_to_duckdb_time(s: &str) -> Result<duckdb::types::Value> {
use chrono::Timelike;
let time = chrono::NaiveTime::parse_from_str(s, "%H:%M:%S").unwrap();
Ok(duckdb::types::Value::Time64(
TimeUnit::Microsecond,
time.num_seconds_from_midnight() as i64,
))
}
struct ParsedAttachDbResource<'a> {
resource_path: &'a str,
name: &'a str,
db_type: &'a str,
extra_args: Option<&'a str>,
}
fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource<'a>> {
lazy_static::lazy_static! {
static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap();
}
for cap in RE.captures_iter(query) {
if let (Some(resource_path), Some(name), Some(db_type)) =
(cap.get(1), cap.get(2), cap.get(3))
{
let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim());
return Some(ParsedAttachDbResource {
resource_path: query[resource_path.start()..resource_path.end()].trim(),
name: query[name.start()..name.end()].trim(),
db_type: query[db_type.start()..db_type.end()].trim(),
extra_args,
});
}
}
None
}
async fn transform_attach_db_resource_query(
parsed: &ParsedAttachDbResource<'_>,
job_id: &Uuid,
client: &AuthedClient,
) -> Result<Vec<String>> {
match parsed.db_type.to_lowercase().as_str() {
"postgres" => {
let resource: PgDatabase = client
.get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
.await?;
let attach_str = format!(
"ATTACH 'dbname={} {} host={} {} {}' AS {} (TYPE postgres{});",
resource.dbname,
resource
.user
.map(|u| format!("user={}", u))
.unwrap_or_default(),
resource.host,
resource
.password
.map(|p| format!("password={}", p))
.unwrap_or_default(),
resource
.port
.map(|p| format!("port={}", p))
.unwrap_or_default(),
parsed.name,
parsed.extra_args.unwrap_or("")
);
Ok(vec![
"INSTALL postgres;".to_string(),
"LOAD postgres;".to_string(),
attach_str,
])
}
"mysql" => {
#[cfg(not(feature = "mysql"))]
return Err(Error::ExecutionErr(
"MySQL feature is not enabled".to_string(),
));
#[cfg(feature = "mysql")]
{
let resource: MysqlDatabase = client
.get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
.await?;
let attach_str = format!(
"ATTACH 'database={} host={} ssl_mode={} {} {} {}' AS {} (TYPE mysql{});",
resource.database,
resource.host,
resource
.ssl
.map(|ssl| if ssl { "required" } else { "disabled" })
.unwrap_or("preferred"),
resource
.password
.map(|p| format!("password={}", p))
.unwrap_or_default(),
resource
.port
.map(|p| format!("port={}", p))
.unwrap_or_default(),
resource
.user
.map(|u| format!("user={}", u))
.unwrap_or_default(),
parsed.name,
parsed.extra_args.unwrap_or("")
);
Ok(vec![
"INSTALL mysql;".to_string(),
"LOAD mysql;".to_string(),
attach_str,
])
}
}
"bigquery" => {
let resource: Value = client
.get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
.await?;
// duckdb's bigquery extension requires a json file as credentials
let bq_credentials_path = make_bq_credentials_path(job_id);
env::set_var("GOOGLE_APPLICATION_CREDENTIALS", &bq_credentials_path);
tokio::fs::write(&bq_credentials_path, resource.to_string())
.await
.map_err(|e| {
Error::ExecutionErr(format!(
"Failed to write BigQuery credentials to {}: {}",
&bq_credentials_path, e
))
})?;
let project_id: String = serde_json::from_value(
resource
.get("project_id")
.ok_or_else(|| {
Error::ExecutionErr("BigQuery resource must contain project_id".to_string())
})?
.to_owned(),
)
.map_err(|_e| Error::ExecutionErr("failed project_id deserialize".to_string()))?;
let attach_str = format!(
"ATTACH 'project={}' as {} (TYPE bigquery{});",
project_id,
parsed.name,
parsed.extra_args.unwrap_or("")
)
.to_string();
Ok(vec![
"INSTALL bigquery FROM community;".to_string(),
"LOAD bigquery;".to_string(),
attach_str,
])
}
_ => Err(Error::ExecutionErr(format!(
"Unsupported db type in DuckDB ATTACH: {}",
parsed.db_type
))),
}
}
// Returns the transformed query and the set of storages used
async fn transform_s3_uris(
query: &str,
client: &AuthedClient,
) -> Result<(
Option<String>,
HashMap<Option<String>, DuckdbConnectionSettingsResponse>,
)> {
let mut transformed_query = None;
lazy_static::lazy_static! {
static ref RE: regex::Regex = regex::Regex::new(r"'s3://([^'/]*)/([^']+)'").unwrap();
}
let mut used_storages = HashMap::new();
for cap in RE.captures_iter(query) {
if let (storage, Some(s3_path)) = (cap.get(1), cap.get(2)) {
let s3_path = s3_path.as_str();
let storage = match storage.map(|m| m.as_str()) {
Some("") | None => None,
Some(s) => Some(s.to_string()),
};
let original_str_lit =
format!("'s3://{}/{}'", storage.as_deref().unwrap_or(""), s3_path);
let duckdb_conn_settings = client
.get_duckdb_connection_settings(&DuckdbConnectionSettingsQueryV2 {
s3_resource_path: None,
storage: storage.clone(),
})
.await?;
let url = match &duckdb_conn_settings {
DuckdbConnectionSettingsResponse { s3_bucket: Some(bucket), .. } => {
format!("'s3://{bucket}/{s3_path}'")
}
DuckdbConnectionSettingsResponse { azure_container_path: Some(base), .. } => {
format!("'{base}/{s3_path}'")
}
_ => {
return Err(Error::ExecutionErr(
"DuckDB connection settings response must have either s3_bucket or azure_container_path".to_string(),
))?;
}
};
transformed_query = Some(
transformed_query
.unwrap_or(query.to_string())
.replace(&original_str_lit, &url),
);
used_storages.insert(storage, duckdb_conn_settings);
}
}
Ok((transformed_query, used_storages))
}
// BigQuery extension requires a json file as credentials
// The file path is set as an env var by do_duckdb
// It is created by transform_attach_db_resource_query (when bigquery is detected)
// and deleted by do_duckdb after the query is executed
fn make_bq_credentials_path(job_id: &Uuid) -> String {
format!("/tmp/service-account-credentials-{}.json", job_id)
}
// duckdb-rs does not support named parameters,
// and it raises an error when passing unused arguments. We cannot prepare batch statements
// but only single SQL statements so it doesn't work when all arguments are not used by
// every single statement.
fn interpolate_named_args<'a>(
query: &str,
args: &'a HashMap<String, duckdb::types::Value>,
) -> (String, Vec<&'a duckdb::types::Value>) {
let mut query = query.to_string();
let mut values = vec![];
for (arg_name, arg_value) in args {
let pat = format!("${}", arg_name);
if !query.contains(&pat) {
continue;
}
values.push(arg_value);
query = query.replace(&pat, &format!("${}", values.len()));
}
(query, values)
}
fn trunc_sig(query: &str) -> &str {
let idx = query.rfind("-- $").unwrap_or(query.len());
// find next \n starting from idx and return everything after it
let idx = query[idx..].find('\n').map(|i| i + idx).unwrap_or(0);
&query[idx..]
}
backend/windmill-worker/src/lib.rs
+2
View File
@@ -20,6 +20,8 @@ mod csharp_executor;
#[cfg(feature = "enterprise")]
mod dedicated_worker;
mod deno_executor;
#[cfg(feature = "duckdb")]
mod duckdb_executor;
mod global_cache;
mod go_executor;
mod graphql_executor;
backend/windmill-worker/src/mysql_executor.rs
+7 -7
View File
@@ -32,13 +32,13 @@ use crate::{
};
#[derive(Deserialize)]
struct MysqlDatabase {
host: String,
user: Option<String>,
password: Option<String>,
port: Option<u16>,
database: String,
ssl: Option<bool>,
pub struct MysqlDatabase {
pub host: String,
pub user: Option<String>,
pub password: Option<String>,
pub port: Option<u16>,
pub database: String,
pub ssl: Option<bool>,
}
fn do_mysql_inner<'a>(
backend/windmill-worker/src/pg_executor.rs
+8 -8
View File
@@ -47,14 +47,14 @@ use lazy_static::lazy_static;
use urlencoding::encode;
use windmill_common::client::AuthedClient;
#[derive(Deserialize)]
struct PgDatabase {
host: String,
user: Option<String>,
password: Option<String>,
port: Option<u16>,
sslmode: Option<String>,
dbname: String,
root_certificate_pem: Option<String>,
pub struct PgDatabase {
pub host: String,
pub user: Option<String>,
pub password: Option<String>,
pub port: Option<u16>,
pub sslmode: Option<String>,
pub dbname: String,
pub root_certificate_pem: Option<String>,
}
lazy_static! {
backend/windmill-worker/src/windmill-client.js
+2
View File
@@ -2998,6 +2998,8 @@ var $RawScript = {
"mssql",
"graphql",
"nativets",
"duckdb",
// for related places search: ADD_NEW_LANG
],
},
path: {
backend/windmill-worker/src/worker.rs
+28
View File
@@ -145,6 +145,9 @@ use crate::ansible_executor::handle_ansible_job;
#[cfg(feature = "mysql")]
use crate::mysql_executor::do_mysql;
#[cfg(feature = "duckdb")]
use crate::duckdb_executor::do_duckdb;
#[cfg(feature = "oracledb")]
use crate::oracledb_executor::do_oracledb;
@@ -2614,6 +2617,30 @@ async fn handle_code_execution_job(
)
.await;
}
} else if language == Some(ScriptLang::DuckDb) {
#[allow(unreachable_code)]
#[cfg(not(feature = "duckdb"))]
{
return Err(Error::internal_err(
"Duck DB requires the duckdb feature to be enabled".to_string(),
));
}
#[cfg(feature = "duckdb")]
{
return do_duckdb(
job,
&client,
&code,
conn,
mem_peak,
canceled_by,
worker_name,
column_order,
occupancy_metrics,
)
.await;
}
} else if language == Some(ScriptLang::Graphql) {
return do_graphql(
job,
@@ -3021,6 +3048,7 @@ fn parse_sig_of_lang(
ScriptLang::Snowflake => Some(windmill_parser_sql::parse_snowflake_sig(code)?),
ScriptLang::Graphql => None,
ScriptLang::Mssql => Some(windmill_parser_sql::parse_mssql_sig(code)?),
ScriptLang::DuckDb => Some(windmill_parser_sql::parse_duckdb_sig(code)?),
ScriptLang::OracleDB => Some(windmill_parser_sql::parse_oracledb_sig(code)?),
#[cfg(feature = "php")]
ScriptLang::Php => Some(windmill_parser_php::parse_php_signature(
cli/bootstrap/script_bootstrap.ts
+2 -1
View File
@@ -76,6 +76,7 @@ func main() (interface{}, error) {
bash: `echo "Hello world"
`,
duckdb: `SELECT 'Hello world' AS message`,
oracledb: `SELECT 'Hello world' AS message`,
powershell: `Write-Output "Hello world"`,
@@ -126,5 +127,5 @@ public class Main {
}
}
`,
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
};
cli/build.sh
+1
View File
@@ -1,3 +1,4 @@
#!/bin/bash
# Note for mac OS users: you need to install gnu-sed with `brew install gnu-sed` and use `gsed` instead of `sed`.
./gen_wm_client.sh
deno run -A dnt.ts
cli/metadata.ts
+3
View File
@@ -528,6 +528,9 @@ export async function inferSchema(
{ name: "database", typ: { resource: "postgresql" } },
...inferedSchema.args,
];
} else if (language === "duckdb") {
const { parse_sql } = await import("./wasm/regex/windmill_parser_wasm.js");
inferedSchema = JSON.parse(parse_sql(content));
} else if (language === "graphql") {
const { parse_graphql } = await import(
"./wasm/regex/windmill_parser_wasm.js"
cli/script.ts
+6 -4
View File
@@ -547,6 +547,8 @@ export function filePathExtensionFromContentType(
return ".my.sql";
} else if (language === "bigquery") {
return ".bq.sql";
} else if (language === "duckdb") {
return ".duckdb.sql";
} else if (language === "oracledb") {
return ".odb.sql";
} else if (language === "snowflake") {
@@ -573,7 +575,7 @@ export function filePathExtensionFromContentType(
return ".nu";
} else if (language === "java") {
return ".java";
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
} else {
throw new Error("Invalid language: " + language);
}
@@ -593,6 +595,7 @@ export const exts = [
".odb.sql",
".sf.sql",
".ms.sql",
".duckdb.sql",
".sql",
".gql",
".ps1",
@@ -601,9 +604,8 @@ export const exts = [
".cs",
".nu",
".playbook.yml",
".java"
// for related places search: ADD_NEW_LANG
".java",
// for related places search: ADD_NEW_LANG
];
export function removeExtensionToPath(path: string): string {
cli/script_common.ts
+5 -2
View File
@@ -9,6 +9,7 @@ export type ScriptLanguage =
| "postgresql"
| "mysql"
| "bigquery"
| "duckdb"
| "oracledb"
| "snowflake"
| "mssql"
@@ -19,7 +20,7 @@ export type ScriptLanguage =
| "nu"
| "ansible"
| "java";
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
export function inferContentTypeFromFilePath(
contentPath: string,
@@ -43,6 +44,8 @@ export function inferContentTypeFromFilePath(
return "bigquery";
} else if (contentPath.endsWith(".odb.sql")) {
return "oracledb";
} else if (contentPath.endsWith(".duckdb.sql")) {
return "duckdb";
} else if (contentPath.endsWith(".sf.sql")) {
return "snowflake";
} else if (contentPath.endsWith(".ms.sql")) {
@@ -67,7 +70,7 @@ export function inferContentTypeFromFilePath(
return "nu";
} else if (contentPath.endsWith(".java")) {
return "java";
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
} else {
throw new Error(
"Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))
cli/sync.ts
+1
View File
@@ -344,6 +344,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner {
else if (language == "nu") ext = "nu";
else if (language == "ansible") ext = "playbook.yml";
else if (language == "java") ext = "java";
else if (language == "duckdb") ext = "duckdb.sql";
// for related places search: ADD_NEW_LANG
else ext = "no_ext";
cli/wasm/regex/windmill_parser_wasm.js
+1
View File
@@ -1,5 +1,6 @@
let WASM_VECTOR_LEN = 0;
let cachedUint8ArrayMemory0 = null;
frontend/package-lock.json
Generated
+4 -4
View File
@@ -74,7 +74,7 @@
"windmill-parser-wasm-nu": "^1.474.1",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.477.1",
"windmill-parser-wasm-regex": "^1.481.0",
"windmill-parser-wasm-regex": "^1.492.1",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.486.1",
"windmill-parser-wasm-yaml": "^1.429.0",
@@ -12837,9 +12837,9 @@
"integrity": "sha512-EY3mSMWpqFPzd7fsLg2/hSfQFU8HpW9nplFwm4JHHCDbcTpBzlvzjPJoHAAGO5kMzowAxjqi5ai/mXjeUWuiSg=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.481.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.481.0.tgz",
"integrity": "sha512-QY9NOaF0M55SBEbbcAkHoXppjdS2gb2NlwqhlZ1ciYWTtIrbZoL5K8xoG4FbB5vWRHxInG1aRdA2tb+JP180yA=="
"version": "1.492.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.492.1.tgz",
"integrity": "sha512-CBdjz3x00z2xA4BRZv8/hsuHcpZgL28UKTjYVSj5O9XMYlY9g0LNsEQ5TTl/oVyFBs62/CPyvWwGvW4oSAObzg=="
},
"node_modules/windmill-parser-wasm-rust": {
"version": "1.429.0",
frontend/package.json
+1 -1
View File
@@ -143,7 +143,7 @@
"windmill-parser-wasm-nu": "^1.474.1",
"windmill-parser-wasm-php": "^1.429.0",
"windmill-parser-wasm-py": "^1.477.1",
"windmill-parser-wasm-regex": "^1.481.0",
"windmill-parser-wasm-regex": "^1.492.1",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.486.1",
"windmill-parser-wasm-yaml": "^1.429.0",
frontend/src/lib/components/Editor.svelte
+4 -1
View File
@@ -133,6 +133,7 @@
import EditorTheme from './EditorTheme.svelte'
import {
BIGQUERY_TYPES,
DUCKDB_TYPES,
MSSQL_TYPES,
MYSQL_TYPES,
ORACLEDB_TYPES,
@@ -482,7 +483,9 @@
? MSSQL_TYPES
: scriptLang === 'oracledb'
? ORACLEDB_TYPES
: []
: scriptLang === 'duckdb'
? DUCKDB_TYPES
: []
).map((t) => ({
label: t,
kind: languages.CompletionItemKind.Function,
frontend/src/lib/components/HighlightCode.svelte
+5 -3
View File
@@ -55,6 +55,8 @@
return sql
case 'oracledb':
return sql
case 'duckdb':
return sql
case 'powershell':
return powershell
case 'php':
@@ -66,10 +68,10 @@
case 'nu':
return python
case 'ansible':
return yaml;
return yaml
case 'java':
return java;
// for related places search: ADD_NEW_LANG
return java
// for related places search: ADD_NEW_LANG
default:
return typescript
}
frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte
+6 -3
View File
@@ -22,6 +22,7 @@
import CSharpIcon from '$lib/components/icons/CSharpIcon.svelte'
import NuIcon from '$lib/components/icons/NuIcon.svelte'
import JavaIcon from '$lib/components/icons/JavaIcon.svelte'
import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte'
export let lang:
| SupportedLanguage
@@ -49,6 +50,7 @@
postgresql: 'Postgresql',
bigquery: 'BigQuery',
oracledb: 'Oracle Database',
duckdb: 'DuckDB',
snowflake: 'Snowflake',
mysql: 'MySQL',
mssql: 'MS SQL Server',
@@ -59,7 +61,7 @@
csharp: 'C#',
nu: 'Nu',
java: 'Java'
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
}
const langToComponent: Record<
@@ -91,8 +93,9 @@
ansible: AnsibleIcon,
csharp: CSharpIcon,
nu: NuIcon,
java: JavaIcon
// for related places search: ADD_NEW_LANG
java: JavaIcon,
duckdb: DuckDbIcon
// for related places search: ADD_NEW_LANG
}
let subIconScale = width === 30 ? 0.6 : 0.8
frontend/src/lib/components/icons/DuckDbIcon.svelte
+19
View File
@@ -0,0 +1,19 @@
<script>
export let width = 16
export let height = 16
</script>
<svg {width} {height} viewBox="0 0 335 335" fill="none" xmlns="http://www.w3.org/2000/svg">
<path
d="M167.2 334.4C74.7 334.4 0 259.7 0 167.2C0 74.7 74.7 0 167.2 0C259.7 0 334.4 74.7 334.4 167.2C334.4 259.7 259.7 334.4 167.2 334.4Z"
fill="black"
/>
<path
d="M256.6 142.4H223.8V191.9H256.6C270.2 191.9 281.5 180.6 281.5 167C281.5 153.3 270.2 142.4 256.6 142.4Z"
fill="#FFF100"
/>
<path
d="M63.1 167.2C63.1 205.4 94.2 236.5 132.4 236.5C170.6 236.5 201.7 205.4 201.7 167.2C201.7 129 170.6 97.9 132.4 97.9C94.2 97.9 63.1 129 63.1 167.2Z"
fill="#FFF100"
/>
</svg>
frontend/src/lib/consts.ts
+50
View File
@@ -203,6 +203,56 @@ export const MSSQL_TYPES = [
'decimal',
'bit'
]
export const DUCKDB_TYPES = [
'BIGINT',
'INT8',
'LONG',
'BIT',
'BITSTRING',
'BLOB',
'BYTEA',
'BINARY',
'VARBINARY',
'BOOLEAN',
'BOOL',
'LOGICAL',
'DATE',
'DECIMAL',
'NUMERIC',
'DOUBLE',
'FLOAT8',
'FLOAT',
'FLOAT4',
'REAL',
'HUGEINT',
'INTEGER',
'INT4',
'INT',
'SIGNED',
'INTERVAL',
'JSON',
'SMALLINT',
'INT2',
'SHORT',
'TIME',
'TIMESTAMP WITH TIME ZONE',
'TIMESTAMPTZ',
'TIMESTAMP',
'DATETIME',
'TINYINT',
'INT1',
'UBIGINT',
'UHUGEINT',
'UINTEGER',
'USMALLINT',
'UTINYINT',
'UUID',
'VARCHAR',
'CHAR',
'BPCHAR',
'TEXT',
'STRING'
]
export const DB_TYPES: Record<DbType, string[]> = {
bigquery: BIGQUERY_TYPES,
frontend/src/lib/infer.ts
+8 -2
View File
@@ -9,6 +9,7 @@ import initRegexParsers, {
parse_sql,
parse_mysql,
parse_oracledb,
parse_duckdb,
parse_bigquery,
parse_snowflake,
parse_graphql,
@@ -96,7 +97,9 @@ export async function inferArgs(
}
let inlineDBResource: string | undefined = undefined
if (['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '')) {
if (
['postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'oracledb'].includes(language ?? '')
) {
await initWasmRegex()
inlineDBResource = parse_db_resource(code)
}
@@ -147,6 +150,9 @@ export async function inferArgs(
...inferedSchema.args
]
}
} else if (language == 'duckdb') {
await initWasmRegex()
inferedSchema = JSON.parse(parse_duckdb(code))
} else if (language == 'snowflake') {
inferedSchema = JSON.parse(parse_snowflake(code))
if (inlineDBResource === undefined) {
@@ -194,7 +200,7 @@ export async function inferArgs(
} else if (language == 'java') {
await initWasmJava()
inferedSchema = JSON.parse(parse_java(code))
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
} else {
return null
}
frontend/src/lib/inferArgSig.ts
+9 -11
View File
@@ -5,23 +5,21 @@ export function argSigToJsonSchemaType(
| string
| { resource: string | null }
| {
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null
}
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null
}
| { dynselect: string }
| { str: string[] | null }
| { object: { key: string; typ: any }[] }
| {
oneof: [
{
oneof: {
label: string
properties: { key: string; typ: any }[]
}
]
},
}[]
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: '' }
frontend/src/lib/script_helpers.ts
+18
View File
@@ -307,6 +307,19 @@ INSERT INTO demo VALUES (@P1, @P2);
UPDATE demo SET col2 = @P3 WHERE col2 = @P2;
`
const DUCKDB_INIT_CODE = `-- $friends_csv (s3object)
-- $name (text) = Ben
ATTACH '$res:u/demo/amazed_postgresql' AS db (TYPE postgres);
CREATE TABLE IF NOT EXISTS db.public.friends (name text);
INSERT INTO db.public.friends
SELECT name FROM read_csv($friends_csv);
SELECT 'Hello ' || $name || ', you have ' ||
(SELECT COUNT(*) FROM read_csv($friends_csv)) || ' new friends !';
`
const GRAPHQL_INIT_CODE = `query($name4: String, $name2: Int, $name3: [String]) {
demo(name1: $name1, name2: $name2, name3: $name3) {
name1,
@@ -1134,6 +1147,9 @@ export const INITIAL_CODE = {
mssql: {
script: MSSQL_INIT_CODE
},
duckdb: {
script: DUCKDB_INIT_CODE
},
graphql: {
script: GRAPHQL_INIT_CODE
},
@@ -1258,6 +1274,8 @@ export function initialCode(
return INITIAL_CODE.mssql.script
} else if (language == 'graphql') {
return INITIAL_CODE.graphql.script
} else if (language == 'duckdb') {
return INITIAL_CODE.duckdb.script
} else if (language == 'php') {
return INITIAL_CODE.php.script
} else if (language == 'rust') {
frontend/src/lib/scripts.ts
+5 -2
View File
@@ -38,6 +38,8 @@ export function scriptLangToEditorLang(
return 'sql'
} else if (lang == 'mssql') {
return 'sql'
} else if (lang == 'duckdb') {
return 'sql'
} else if (lang == 'python3') {
return 'python'
} else if (lang == 'bash') {
@@ -140,7 +142,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string]
['csharp', 'C#'],
['docker', 'Docker'],
['nu', 'Nu'],
['java', 'Java']
['java', 'Java'],
['duckdb', 'DuckDB']
// for related places search: ADD_NEW_LANG
]
export function processLangs(selected: string | undefined, langs: string[]): string[] {
@@ -150,7 +153,7 @@ export function processLangs(selected: string | undefined, langs: string[]): str
let ls = langs.filter((lang) => lang !== 'nativets')
//those languages are newer and may not be in the saved list
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java']
let nl = ['bunnative', 'rust', 'ansible', 'csharp', 'nu', 'java', 'duckdb']
// for related places search: ADD_NEW_LANG
nl.forEach((lang) => {
if (!ls.includes(lang)) {
frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte
+1 -1
View File
@@ -755,7 +755,7 @@
<Badge color="blue">priority: {job.priority}</Badge>
</div>
{/if}
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'dependency'].includes(job.tag)}
{#if job.tag && !['deno', 'python3', 'flow', 'other', 'go', 'postgresql', 'mysql', 'bigquery', 'snowflake', 'mssql', 'graphql', 'oracledb', 'nativets', 'bash', 'powershell', 'php', 'rust', 'other', 'ansible', 'csharp', 'nu', 'java', 'duckdb', 'dependency'].includes(job.tag)}
<!-- for related places search: ADD_NEW_LANG -->
<div>
<Badge color="indigo">Tag: {job.tag}</Badge>