Specific asset tables (#7323)

* data tables settings ui

* install runed

* zod 4 fixes

* use new toJSONSchema

* Migrate ducklake catalogs to more generic custom instance databases

* fix compilation

* Safety conversion for old duckdb ffi

* data tables settings

* ts client basis

* inline run works

* datatables work

* Revert "datatables work"

This reverts commit 6e1588d59e.

* datatables work (without leaking pg credentials)

* println

* separate sqlUtils.ts

* nit

* Separate custom instance db Select and Wizard components

* nit

* nit wording

* add tags to custom instance dbs

* error when trying to use ducklake as datatable or opposite

* show status in dropdown

* data table instance setup works

* sqk function for ducklake

* factorize logic

* fix temp reactivity

* Data table assetexplore

* Migrate S3 permissions to modal

* Revert "Migrate S3 permissions to modal"

This reverts commit 0631d03cb0.

* nit query -> fetch

* Custom instance setup new look

* run_language_executor separate fn

* run_inline param

* nit wording

* Better typed client

* Data tables display as assets in frontend

* asset db icon

* nit

* cleaner errors

* nit

* Fix sed calls in mac

* run_inline_script_preview in python client

* basic python datatable client

* datatable and datalake parser in python

* ducklake client python

* nit fix

* Fix migration producing NULL instead of {} when no custom databases

* merge conflict fail

* python ducklake client arg fix

* parse or infer sql types in ts client

* ts asset parser, detect datatable & ducklake R/W

* fix sql repl for other read ops than select

* export type SqlTemplateFunction

* rename list_custom_instance_pg_databases

* typecheck datatable and ducklake name in Typescript

* Fix typecheck datatable and ducklake in TS

* declare module overriding instead of extending

* infer_sql_type in python client

* SqlQuery object in python

* fix merge conflicts

* update const_format

* CI fix

* factor out to var_identifiers

* sqlx prepare

* unnecessary security (admin is required)

* clearer comment

* ee repo ref

* nit snake case

* claude step 1: detect var declarations

* move detect_sql_access_type to common mod

* claude step 2: detect when saved vars are queried

* Revert "claude step 2: detect when saved vars are queried"

This reverts commit 1e1f930568.

* Revert "claude step 1: detect var declarations"

This reverts commit f866f4819d.

* remove ducklake/datatable and default

* detect data table assigns in var_identifiers

* Python parser successfully infers R/W/RW from ducklake / datatable

* still register ducklake/datatable if not used as unknown R/W

* Go to settings button in Assets Dropdown on not found

* nit

* sqlx prepare fail

* manual fix, somehow sqlx prepare won't do it

* fix frontend ci

* ee repo ref

* ducklake_user doesnt exist in unit tests

* nit fix

* ui nit

* nit

* nit missing clone

* fork ducklakes and datatables

* fix surface hover bug

* stupid mistake

* better deeply reactive mutable derived

* Ducklake picker

* Editor bar data tables

* DuckDB supports datatables

* datatable in duckdb asset parser

* duckdb asset parser var_identifiers

* Revert "duckdb asset parser var_identifiers"

This reverts commit 88068b1a77.

* sqlx prepare

* Box pin in test_workflow_as_code to fix stack overflow

* stash

* sql asset parser parses most s3 literals

* nit

* Detect attach + handle returning RW

* detect assets used with dot notation

* detect implicit access with USE dl; syntax

* Add assets as unknown if var was never used

* Support default ducklake/datatable main in parser

* ignore asset parsing errors in frontend (avoid flow layout shift)

* super weird duplication (merge conflict ?)

* nits

* fix duckdb parser detecting too much as asset when RW ctx is unknown

* fix transparent assets btn

* missing arg

* nit styling

* asset parser specific table parsing

* fix resource specific table parsing

* More concise asset display in flows + better icons

* fix assets page filtering out resources with added table

* Fix frontend to support specific table assets

* Open DB Manager to specific table

* Specific table parser in Python and TS + unit tests

* Fix UPDATE setting access to None

* fix flow edge rendering on top of output picker

* python parser fix var override bug

* add ts test

* fix compilation

* sqlx prepare

* update parsers version

* fix missing schema key onDelete
This commit is contained in:
Diego Imbert
2025-12-11 14:34:17 +01:00
committed by GitHub
parent 50c840a41b
commit 66c62e088c
54 changed files with 1293 additions and 572 deletions
@@ -29,8 +29,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -39,8 +39,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -15,7 +15,7 @@
]
},
"nullable": [
null
true
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
@@ -239,8 +239,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -39,8 +39,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -154,8 +154,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Varchar"
]
},
"nullable": []
},
"hash": "99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a"
}
@@ -29,8 +29,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -184,8 +184,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT progress::bigint FROM tutorial_progress WHERE email = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "progress",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2"
}
@@ -154,8 +154,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -121,8 +121,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -104,8 +104,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', asset.path,\n 'kind', asset.kind,\n 'usages', ARRAY_AGG(jsonb_build_object(\n 'path', asset.usage_path,\n 'kind', asset.usage_kind,\n 'access_type', asset.usage_access_type\n )),\n 'metadata', (CASE\n WHEN asset.kind = 'resource' THEN\n jsonb_build_object('resource_type', resource.resource_type)\n ELSE\n NULL\n END\n )\n )) as \"list!: _\"\n FROM asset\n LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1\n WHERE asset.workspace_id = $1\n AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)\n AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))\n AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1))\n GROUP BY asset.path, asset.kind, resource.resource_type\n ORDER BY asset.path, asset.kind",
"query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', asset.path,\n 'kind', asset.kind,\n 'usages', ARRAY_AGG(jsonb_build_object(\n 'path', asset.usage_path,\n 'kind', asset.usage_kind,\n 'access_type', asset.usage_access_type\n )),\n 'metadata', (CASE\n WHEN asset.kind = 'resource' THEN\n jsonb_build_object('resource_type', resource.resource_type)\n ELSE\n NULL\n END\n )\n )) as \"list!: _\"\n FROM asset\n LEFT JOIN resource ON asset.kind = 'resource'\n AND array_to_string((string_to_array(asset.path, '/'))[1:3], '/') = resource.path -- With specific table, asset path can be e.g u/diego/pg_db/table_name\n AND resource.workspace_id = $1\n WHERE asset.workspace_id = $1\n AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)\n AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))\n AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1))\n GROUP BY asset.path, asset.kind, resource.resource_type\n ORDER BY asset.path, asset.kind",
"describe": {
"columns": [
{
@@ -18,5 +18,5 @@
null
]
},
"hash": "92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6"
"hash": "c94cd50ff1233025b170efee6489e6637676a8b0435a4612a7efbfff6ea2543d"
}
@@ -184,8 +184,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
@@ -104,8 +104,7 @@
"postgres",
"sqs",
"gcp",
"mqtt",
"nextcloud"
"mqtt"
]
}
}
+18 -15
View File
@@ -2822,7 +2822,7 @@ dependencies = [
"parquet",
"rand 0.8.5",
"regex",
"sqlparser",
"sqlparser 0.55.0",
"tempfile",
"tokio",
"url",
@@ -2899,7 +2899,7 @@ dependencies = [
"parquet",
"paste",
"recursive",
"sqlparser",
"sqlparser 0.55.0",
"tokio",
"web-time",
]
@@ -3075,7 +3075,7 @@ dependencies = [
"paste",
"recursive",
"serde_json",
"sqlparser",
"sqlparser 0.55.0",
]
[[package]]
@@ -3371,7 +3371,7 @@ dependencies = [
"log",
"recursive",
"regex",
"sqlparser",
"sqlparser 0.55.0",
]
[[package]]
@@ -8602,15 +8602,6 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "nom"
version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
dependencies = [
"memchr",
]
[[package]]
name = "notify"
version = "6.1.1"
@@ -12264,6 +12255,17 @@ dependencies = [
"sqlparser_derive",
]
[[package]]
name = "sqlparser"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f"
dependencies = [
"log",
"recursive",
"sqlparser_derive",
]
[[package]]
name = "sqlparser_derive"
version = "0.3.0"
@@ -15184,7 +15186,6 @@ dependencies = [
"lazy_static",
"libloading 0.8.9",
"memchr",
"nom 8.0.0",
"object_store",
"once_cell",
"pep440_rs",
@@ -15642,6 +15643,7 @@ dependencies = [
"rustpython-parser",
"serde_json",
"windmill-parser",
"windmill-parser-sql",
]
[[package]]
@@ -15705,11 +15707,11 @@ version = "1.591.2"
dependencies = [
"anyhow",
"lazy_static",
"nom 8.0.0",
"regex",
"regex-lite",
"serde",
"serde_json",
"sqlparser 0.59.0",
"windmill-parser",
]
@@ -15729,6 +15731,7 @@ dependencies = [
"triomphe",
"wasm-bindgen",
"windmill-parser",
"windmill-parser-sql",
]
[[package]]
-2
View File
@@ -158,7 +158,6 @@ kube.workspace = true
k8s-openapi.workspace = true
libloading.workspace = true
bitflags.workspace = true
nom.workspace = true
globset.workspace = true
@@ -378,7 +377,6 @@ pg_escape = "0.1.1"
async-nats = "0.38.0"
nkeys = "0.4.4"
nu-parser = { version = "0.101.0", default-features = false }
nom = "8.0.0"
globset = "0.4.16"
process-wrap = { version = "8.2.1", features = ["tokio1"] }
@@ -10,6 +10,7 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
rustpython-parser.workspace = true
itertools.workspace = true
serde_json.workspace = true
@@ -2,12 +2,12 @@ use rustpython_ast::{Constant, Expr, ExprConstant, Visitor};
use rustpython_parser::{ast::Suite, Parse};
use std::collections::HashMap;
use windmill_parser::asset_parser::{
detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
ParseAssetsResult,
};
use AssetUsageAccessType::*;
pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>>> {
pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
let ast = Suite::parse(input, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
@@ -15,17 +15,13 @@ pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>
ast.into_iter()
.for_each(|stmt| assets_finder.visit_stmt(stmt));
for (kind, name) in assets_finder.var_identifiers.into_values() {
for (kind, path) in assets_finder.var_identifiers.into_values() {
// if a db = wmill.datatable() was never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
if assets_finder
.assets
.iter()
.all(|a| !(a.kind == kind && a.path == name))
{
if asset_was_used(&assets_finder.assets, (kind, &path)) == false {
assets_finder
.assets
.push(ParseAssetsResult { kind, access_type: None, path: name });
.push(ParseAssetsResult { kind, access_type: None, path });
}
}
@@ -33,7 +29,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>
}
struct AssetsFinder {
assets: Vec<ParseAssetsResult<String>>,
assets: Vec<ParseAssetsResult>,
var_identifiers: HashMap<String, (AssetKind, String)>,
}
@@ -41,41 +37,27 @@ impl Visitor for AssetsFinder {
// Handle assignment statements like: x = wmill.datatable('name')
fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) {
// Check if the value is a call to a tracked function
if let Some((kind, name)) = self.extract_asset_from_call(&node.value) {
// Track all target variables
for target in &node.targets {
if let Expr::Name(name_expr) = target {
let Ok(var_name) = name_expr.id.parse::<String>();
self.var_identifiers
.insert(var_name, (kind.clone(), name.clone()));
}
}
} else {
// If not wmill.datatable or similar, remove any tracked variables
// It means the identifier is no longer refering to an asset
for target in &node.targets {
if let Expr::Name(name_expr) = target {
let Ok(var_name) = name_expr.id.parse::<String>();
let removed = self.var_identifiers.remove(&var_name);
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
match removed {
Some((kind, name)) => {
if self
.assets
.iter()
.all(|a| !(a.kind == kind && a.path == name))
{
self.assets.push(ParseAssetsResult {
kind,
access_type: None,
path: name,
});
}
}
None => {}
if let Some(Expr::Name(expr_name)) = node.targets.first() {
// Remove any tracked variables with that name in case of reassignment
let Ok(var_name) = expr_name.id.parse::<String>();
let removed = self.var_identifiers.remove(&var_name);
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
match removed {
Some((kind, path)) => {
if !asset_was_used(&self.assets, (kind, &path)) {
self.assets
.push(ParseAssetsResult { kind, access_type: None, path });
}
}
None => {}
}
if let Some((kind, name)) = self.extract_asset_from_call(&node.value) {
// Track target variable
let Ok(var_name) = expr_name.id.parse::<String>();
self.var_identifiers
.insert(var_name, (kind.clone(), name.clone()));
}
}
// Continue with generic visit to catch any other assets in the expression
@@ -87,7 +69,7 @@ impl Visitor for AssetsFinder {
fn visit_expr_constant(&mut self, node: ExprConstant) {
match node.value {
Constant::Str(s) => {
if let Some((kind, path)) = parse_asset_syntax(&s) {
if let Some((kind, path)) = parse_asset_syntax(&s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
@@ -108,7 +90,7 @@ impl Visitor for AssetsFinder {
if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
&keyword.value
{
if let Some((kind, path)) = parse_asset_syntax(s) {
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
@@ -184,8 +166,8 @@ impl AssetsFinder {
value, ..
}) = node.func.as_ref()
{
if let Expr::Name(name_expr) = value.as_ref() {
name_expr.id.parse().map_err(|_| ())?
if let Expr::Name(expr_name) = value.as_ref() {
expr_name.id.parse().map_err(|_| ())?
} else {
return Err(());
}
@@ -197,26 +179,31 @@ impl AssetsFinder {
// Continue
} else if let Some((kind, ref path)) = self.var_identifiers.get(&obj_name) {
if ident == "query" {
let name_expr = node.args.get(0).or_else(|| {
let expr_name = node.args.get(0).or_else(|| {
node.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some("name"))
.map(|kw| &kw.value)
});
match name_expr {
Some(Expr::Constant(ExprConstant {
value: Constant::Str(sql_query), ..
})) => {
let access_type = detect_sql_access_type(&sql_query);
self.assets.push(ParseAssetsResult {
kind: *kind,
path: path.to_string(),
access_type,
});
return Ok(());
}
let sql = match expr_name {
Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql,
_ => return Err(()),
};
let duckdb_conn_prefix = match kind {
AssetKind::DataTable => "datatable",
AssetKind::Ducklake => "ducklake",
_ => return Ok(()),
};
let sql = format!("ATTACH '{duckdb_conn_prefix}://{path}' AS dt; USE dt; {sql}");
// We use the SQL parser to detect if it's a read or write query
match windmill_parser_sql::parse_assets(&sql) {
Ok(sql_assets) => {
self.assets.extend(sql_assets);
}
_ => {}
}
return Ok(());
} else {
return Err(());
}
@@ -249,7 +236,9 @@ impl AssetsFinder {
match arg_val {
Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value);
let path = parse_asset_syntax(&value, false)
.map(|(_, p)| p)
.unwrap_or(&value);
self.assets
.push(ParseAssetsResult { kind, path: path.to_string(), access_type });
}
@@ -259,5 +248,157 @@ impl AssetsFinder {
}
}
struct Arg(usize, &'static str);
// Positional arguments in python can also be used by their name
struct Arg(usize, &'static str);
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_py_asset_parser_load_s3() {
let input = r#"
import wmill
def main():
wmill.load_s3_file('s3:///test.csv')
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/test.csv".to_string(),
access_type: Some(R)
},])
);
}
#[test]
fn test_py_asset_parser_unused_sql() {
let input = r#"
import wmill
def main():
db = wmill.datatable()
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main".to_string(),
access_type: None
},])
);
}
#[test]
fn test_py_asset_parser_sql_read() {
let input = r#"
import wmill
def main(x: int):
db = wmill.datatable('dt')
return db.query('SELECT * FROM friends WHERE age = $1', x).fetch()
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(R)
},])
);
}
#[test]
fn test_py_asset_parser_sql_read_write() {
let input = r#"
import wmill
def main(x: int):
db = wmill.datatable('dt')
db.query('UPDATE friends SET x = $1', x).fetch()
db.query('SELECT * FROM friends WHERE age = $1', x).fetch_one()
db.query('SELECT * FROM analytics').fetch()
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/analytics".to_string(),
access_type: Some(R)
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(RW)
},
])
);
}
#[test]
fn test_py_asset_parser_multiple_sql_scopes() {
let input = r#"
import wmill
def main():
def f(x: int):
db = wmill.datatable()
return db.query('SELECT * FROM friends WHERE age = $1', x)
db = wmill.datatable('another1')
return db.query('INSERT INTO customers VALUES ($1)', 0)
def g():
db = wmill.ducklake('another2')
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1/customers".to_string(),
access_type: Some(W)
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "another2".to_string(),
access_type: None
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/friends".to_string(),
access_type: Some(R)
},
])
);
}
#[test]
fn test_py_asset_parser_overriden_var_identifier() {
let input = r#"
import wmill
def main():
db = wmill.datatable('another1')
def g():
db = wmill.ducklake()
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1".to_string(),
access_type: None
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
},
])
);
}
}
@@ -20,4 +20,4 @@ anyhow.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde.workspace = true
nom.workspace = true
sqlparser = { version = "0.59.0", features = ["visitor"] }
@@ -1,151 +1,626 @@
use std::collections::HashMap;
use sqlparser::{
ast::{
CopyTarget, Expr, ObjectName, TableFactor, TableObject, Value, ValueWithSpan, Visit,
Visitor,
},
dialect::DuckDbDialect,
parser::Parser,
};
use windmill_parser::asset_parser::{
merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult,
asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
ParseAssetsResult,
};
use AssetUsageAccessType::*;
use nom::{
branch::alt,
bytes::complete::{tag, tag_no_case, take_while},
character::complete::{char, multispace0},
combinator::opt,
sequence::preceded,
IResult, Parser,
};
pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
let statements = Parser::parse_sql(&DuckDbDialect, input)?;
pub fn parse_assets<'a>(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<&str>>> {
let mut assets = Vec::new();
let mut remaining = input;
let mut collector = AssetCollector::new();
for statement in statements {
let _ = statement.visit(&mut collector);
}
while !remaining.trim().is_empty() {
if let Ok((rest, _)) = parse_comment(remaining) {
remaining = rest; // skip comment
}
if let Ok((rest, res)) = parse_asset(remaining) {
assets.push(res);
remaining = rest;
} else {
remaining = &remaining[1..]; // skip 1 char and continue
for (_, (kind, path)) in collector.var_identifiers {
if !asset_was_used(&collector.assets, (kind, &path)) {
collector
.assets
.push(ParseAssetsResult { kind, access_type: None, path: path });
}
}
Ok(merge_assets(assets))
Ok(merge_assets(collector.assets))
}
fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> {
alt((
parse_s3_object_read.map(|path| ParseAssetsResult {
path,
kind: AssetKind::S3Object,
access_type: Some(R),
}),
parse_s3_object_write.map(|path| ParseAssetsResult {
path,
kind: AssetKind::S3Object,
access_type: Some(W),
}),
// Parse ambiguous access_types at the end if we could not find precisely read or copy
parse_s3_object_lit.map(|path| ParseAssetsResult {
path,
kind: AssetKind::S3Object,
access_type: None,
}),
parse_resource_lit.map(|path| ParseAssetsResult {
path,
kind: AssetKind::Resource,
access_type: None,
}),
parse_ducklake_lit.map(|path| ParseAssetsResult {
path,
kind: AssetKind::Ducklake,
access_type: None,
}),
parse_datatable_lit.map(|path| ParseAssetsResult {
path,
kind: AssetKind::DataTable,
access_type: None,
}),
))
.parse(input)
/// Visitor that collects S3 asset literals from SQL statements
struct AssetCollector {
assets: Vec<ParseAssetsResult>,
// e.g set to Read when we are inside a SELECT ... FROM ... statement
current_access_type_stack: Vec<AssetUsageAccessType>,
// e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") }
var_identifiers: HashMap<String, (AssetKind, String)>,
// e.g USE dl;
currently_used_asset: Option<(AssetKind, String)>,
}
/// Any expression that reads an s3 asset
fn parse_s3_object_read(input: &str) -> IResult<&str, &str> {
alt((parse_s3_object_read_fn, parse_s3_object_select_from)).parse(input)
impl AssetCollector {
fn new() -> Self {
Self {
assets: Vec::new(),
current_access_type_stack: Vec::with_capacity(8),
var_identifiers: HashMap::new(),
currently_used_asset: None,
}
}
// Detect when we do 'a.b' and 'a' is associated with an asset in var_identifiers
// Or when we access 'b' and we did USE a;
fn get_associated_asset_from_obj_name(&self, name: &ObjectName) -> Option<ParseAssetsResult> {
let access_type = self.current_access_type_stack.last().copied();
if name.0.len() == 1 {
let ident = name.0.first()?.as_ident()?;
if ident.quote_style.is_some() {
return None;
}
let specific_table = &ident.value;
// We don't want to infer that any simple identifier refers to an asset if
// we are not in a known R/W context
if access_type.is_none() {
return None;
}
if let Some((kind, path)) = &self.currently_used_asset {
let path = format!("{}/{}", path, specific_table);
return Some(ParseAssetsResult { kind: *kind, access_type, path });
}
}
// Check if the first part of the name (the a in a.b) is associated with an asset
if name.0.len() < 2 {
return None;
}
let ident = name.0.first()?.as_ident()?;
let (kind, path) = self.var_identifiers.get(&ident.value)?;
let path = if name.0.len() == 2 {
let specific_table = &name.0.get(1)?.as_ident()?.value;
format!("{}/{}", path, specific_table)
} else {
path.clone()
};
Some(ParseAssetsResult { kind: *kind, access_type, path })
}
fn handle_string_literal(&mut self, s: &str) {
// Check if the string matches our asset syntax patterns
if let Some((kind, path)) = parse_asset_syntax(s, false) {
if kind == AssetKind::S3Object {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: self.current_access_type_stack.last().copied(),
});
}
}
}
fn handle_obj_name_pre(&mut self, name: &ObjectName) {
if let Some(fname) = get_trivial_obj_name(name) {
if is_read_fn(fname) {
self.current_access_type_stack.push(R);
}
}
if let Some(str_lit) = get_str_lit_from_obj_name(name) {
self.handle_string_literal(str_lit);
}
// Writes to tables should be handled directly when visiting the statement
if self.current_access_type_stack.last() == Some(&R) {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
}
fn handle_obj_name_post(&mut self, name: &ObjectName) {
if self.current_access_type_stack.is_empty() {
return;
}
if let Some(fname) = get_trivial_obj_name(name) {
if is_read_fn(fname) {
self.current_access_type_stack.pop();
}
}
}
fn handle_table_with_joins(&mut self, table_with_joins: &sqlparser::ast::TableWithJoins) {
if let TableFactor::Table { name, .. } = &table_with_joins.relation {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
for join in &table_with_joins.joins {
if let TableFactor::Table { name, .. } = &join.relation {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
}
}
}
/// Any expression that writes to an s3 asset
fn parse_s3_object_write(input: &str) -> IResult<&str, &str> {
// COPY (...) TO 's3://...'
let (input, _) = (tag_no_case("TO"), multispace0).parse(input)?;
let (input, path) = parse_s3_object_lit(input)?;
Ok((input, path))
impl Visitor for AssetCollector {
type Break = ();
fn pre_visit_table_factor(
&mut self,
table_factor: &TableFactor,
) -> std::ops::ControlFlow<Self::Break> {
match table_factor {
TableFactor::Table { name, args, .. } => {
if args.is_none() {
// Avoid Table Functions
self.handle_obj_name_pre(name);
}
}
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn post_visit_table_factor(
&mut self,
table_factor: &TableFactor,
) -> std::ops::ControlFlow<Self::Break> {
match table_factor {
TableFactor::Table { name, .. } => self.handle_obj_name_post(name),
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn pre_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow<Self::Break> {
match expr {
Expr::Value(ValueWithSpan { value: Value::SingleQuotedString(s), .. }) => {
self.handle_string_literal(s)
}
Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => {
self.handle_string_literal(s);
}
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn post_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow<Self::Break> {
match expr {
Expr::Function(func) => self.handle_obj_name_post(&func.name),
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn pre_visit_statement(
&mut self,
statement: &sqlparser::ast::Statement,
) -> std::ops::ControlFlow<Self::Break> {
match statement {
sqlparser::ast::Statement::Query(_) => {
// don't forget pop() in post_visit_statement
self.current_access_type_stack.push(R);
}
sqlparser::ast::Statement::Insert(insert) => {
let access_type = if insert.returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
match insert.table {
TableObject::TableName(ref name) => {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
_ => {}
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::Update { returning, table, from, .. } => {
if let Some(from_tables) = from {
let from_tables = match from_tables {
sqlparser::ast::UpdateTableFromKind::AfterSet(tables) => tables,
sqlparser::ast::UpdateTableFromKind::BeforeSet(tables) => tables,
};
self.current_access_type_stack.push(R);
for table_with_joins in from_tables {
self.handle_table_with_joins(table_with_joins);
}
self.current_access_type_stack.pop();
}
let access_type = if returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
self.handle_table_with_joins(table);
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::Delete(delete) => {
let access_type = if delete.returning.is_some() { RW } else { W };
self.current_access_type_stack.push(access_type);
for name in &delete.tables {
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
}
let tables = match &delete.from {
sqlparser::ast::FromTable::WithFromKeyword(tables) => tables,
sqlparser::ast::FromTable::WithoutKeyword(tables) => tables,
};
for table_with_joins in tables {
self.handle_table_with_joins(table_with_joins);
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::CreateTable(create_table) => {
self.current_access_type_stack.push(W);
if let Some(asset) = self.get_associated_asset_from_obj_name(&create_table.name) {
self.assets.push(asset);
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::CreateView { name, .. } => {
self.current_access_type_stack.push(W);
if let Some(asset) = self.get_associated_asset_from_obj_name(name) {
self.assets.push(asset);
}
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => {
self.current_access_type_stack.push(W);
self.handle_string_literal(filename);
self.current_access_type_stack.pop();
}
sqlparser::ast::Statement::AttachDuckDBDatabase {
database_path,
database_alias,
..
} => {
if let Some((kind, path)) = parse_asset_syntax(&database_path.value, true) {
if kind == AssetKind::Ducklake
|| kind == AssetKind::DataTable
|| kind == AssetKind::Resource
{
if let Some(database_alias) = database_alias {
self.var_identifiers
.insert(database_alias.value.clone(), (kind, path.to_string()));
}
}
}
}
sqlparser::ast::Statement::DetachDuckDBDatabase { database_alias, .. } => {
let asset = self.var_identifiers.remove(&database_alias.value);
if self.currently_used_asset == asset {
self.currently_used_asset = None;
}
}
sqlparser::ast::Statement::Use(sqlparser::ast::Use::Object(obj_name)) => {
if let Some((kind, path)) = self.var_identifiers.get(&obj_name.to_string()) {
self.currently_used_asset = Some((*kind, path.clone()));
} else {
self.currently_used_asset = None;
}
}
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn post_visit_statement(
&mut self,
statement: &sqlparser::ast::Statement,
) -> std::ops::ControlFlow<Self::Break> {
match statement {
sqlparser::ast::Statement::Query(_) => {
self.current_access_type_stack.pop();
}
_ => {}
}
std::ops::ControlFlow::Continue(())
}
fn pre_visit_query(
&mut self,
_query: &sqlparser::ast::Query,
) -> std::ops::ControlFlow<Self::Break> {
self.current_access_type_stack.push(R);
std::ops::ControlFlow::Continue(())
}
fn post_visit_query(
&mut self,
_query: &sqlparser::ast::Query,
) -> std::ops::ControlFlow<Self::Break> {
self.current_access_type_stack.pop();
std::ops::ControlFlow::Continue(())
}
// We do not use pre_visit_relation because we cannot know if an ObjectName is a table or a function
}
/// read_parquet('s3://...')
fn parse_s3_object_read_fn(input: &str) -> IResult<&str, &str> {
let (input, _) = alt((
tag_no_case("read_parquet"),
tag_no_case("read_csv"),
tag_no_case("read_json"),
))
.parse(input)?;
let (input, _) = multispace0(input)?;
let (input, _) = char('(')(input)?;
let (input, _) = multispace0(input)?;
let (input, path) = parse_s3_object_lit(input)?;
let (input, _) = multispace0(input)?;
let (input, _) = char(')')(input)?;
Ok((input, path))
fn is_read_fn(fname: &str) -> bool {
fname.eq_ignore_ascii_case("read_parquet")
|| fname.eq_ignore_ascii_case("read_csv")
|| fname.eq_ignore_ascii_case("read_json")
}
/// SELECT ... FROM 's3://...'
fn parse_s3_object_select_from(input: &str) -> IResult<&str, &str> {
let (input, _) = tag_no_case("FROM").parse(input)?;
let (input, _) = multispace0(input)?;
let (input, path) = parse_s3_object_lit(input)?;
Ok((input, path))
}
/// 's3://...'
fn parse_s3_object_lit(input: &str) -> IResult<&str, &str> {
let (input, _) = quote(input)?;
let (input, _) = tag("s3://").parse(input)?;
let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
let (input, _) = quote(input)?;
Ok((input, path))
fn get_trivial_obj_name(name: &sqlparser::ast::ObjectName) -> Option<&str> {
if name.0.len() != 1 {
return None;
}
Some(name.0.first()?.as_ident()?.value.as_str())
}
fn quote(input: &str) -> IResult<&str, char> {
alt((char('\''), char('\"'))).parse(input)
fn get_str_lit_from_obj_name(name: &ObjectName) -> Option<&str> {
if name.0.len() != 1 {
return None;
}
let ident = name.0.first()?.as_ident()?;
if ident.quote_style != Some('\'') {
return None;
}
Some(ident.value.as_str())
}
fn parse_resource_lit(input: &str) -> IResult<&str, &str> {
let (input, _) = quote(input)?;
let (input, _) = alt((tag("$res:"), tag("res://"))).parse(input)?;
let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
let (input, _) = quote(input)?;
Ok((input, path))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_sql_asset_parser_s3_literals() {
let input = r#"
SELECT * FROM read_parquet('s3:///a.parquet');
COPY (SELECT * FROM 's3://snd/b.parquet') TO 's3:///c.parquet';
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/a.parquet".to_string(),
access_type: Some(R)
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/c.parquet".to_string(),
access_type: Some(W)
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "snd/b.parquet".to_string(),
access_type: Some(R)
},
])
);
}
fn parse_ducklake_lit(input: &str) -> IResult<&str, &str> {
let (input, _) = quote(input)?;
let (input, _) = tag("ducklake").parse(input)?;
let (input, path) =
opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?;
let (input, _) = quote(input)?;
Ok((input, path.unwrap_or("main")))
}
#[test]
fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT 2;
USE dl;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl".to_string(),
access_type: None
},])
);
}
fn parse_datatable_lit(input: &str) -> IResult<&str, &str> {
let (input, _) = quote(input)?;
let (input, _) = tag("datatable").parse(input)?;
let (input, path) =
opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?;
let (input, _) = quote(input)?;
Ok((input, path.unwrap_or("main")))
}
#[test]
fn test_sql_asset_parser_attach_dot_notation_read() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
SELECT * FROM dl.table1;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl/table1".to_string(),
access_type: Some(R)
},])
);
}
fn parse_comment(input: &str) -> IResult<&str, &str> {
let (input, _) = tag("--").parse(input)?;
let (input, comment) = take_while(|c| c != '\n')(input)?;
Ok((input, comment))
#[test]
fn test_sql_asset_parser_attach_dot_notation_write() {
let input = r#"
ATTACH 'datatable://my_dt' AS dt;
SELECT dt.read_bait FROM unrelated_table; -- dt. doesn't access the asset
INSERT INTO dt.table1 VALUES ('test');
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "my_dt/table1".to_string(),
access_type: Some(W)
},])
);
}
#[test]
fn test_sql_asset_parser_detach() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
DETACH dl;
SELECT * FROM dl.table1;
"#;
let s = parse_assets(input);
assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![]));
}
#[test]
fn test_sql_asset_parser_implicit_use_asset() {
let input = r#"
ATTACH 'ducklake://my_dl' AS dl;
USE dl;
INSERT INTO table1 VALUES ('test');
USE memory;
SELECT * FROM table1;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "my_dl/table1".to_string(),
access_type: Some(W)
},])
);
}
#[test]
fn test_sql_asset_parser_default_main() {
let input = r#"
ATTACH 'datatable' AS dl;
INSERT INTO dl.table1 VALUES ('test');
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/table1".to_string(),
access_type: Some(W)
},])
);
}
#[test]
fn test_sql_asset_parser_create_table() {
let input = r#"
ATTACH 'ducklake' AS dl; USE dl;
CREATE TABLE friends (
name text,
age int
);
INSERT INTO friends VALUES ($name, $age);
SELECT * FROM friends;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/friends".to_string(),
access_type: Some(RW)
},])
);
}
// Make sure a_function is not detected as main/a_function
#[test]
fn test_sql_asset_parser_function_table() {
let input = r#"
ATTACH 'ducklake' AS dl; USE dl;
SELECT * FROM a_function('');
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
},])
);
}
#[test]
fn test_sql_asset_parser_delete() {
let input = r#"
ATTACH 'ducklake' AS dl;
USE dl;
DELETE FROM table1;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
},])
);
}
#[test]
fn test_sql_asset_parser_update() {
let input = r#"
ATTACH 'ducklake' AS dl;
USE dl;
UPDATE table1 SET id = NULL;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
},])
);
}
#[test]
fn test_sql_asset_parser_resource() {
let input = r#"
ATTACH 'res://u/user/pg_resource' AS db (TYPE postgres);
USE db;
SELECT * FROM table1;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Resource,
path: "u/user/pg_resource/table1".to_string(),
access_type: Some(R)
},])
);
}
#[test]
fn test_sql_asset_parser_update_with_dot_notation() {
let input = r#"
ATTACH 'ducklake' AS dl;
UPDATE dl.table1 SET id = NULL;
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main/table1".to_string(),
access_type: Some(W)
},])
);
}
}
@@ -15,6 +15,7 @@ serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
swc_common.workspace = true
triomphe.workspace = true
swc_ecma_parser.workspace = true
@@ -5,12 +5,12 @@ use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, Str};
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax};
use swc_ecma_visit::{Visit, VisitWith};
use windmill_parser::asset_parser::{
detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType,
ParseAssetsResult,
};
use AssetUsageAccessType::*;
pub fn parse_assets(code: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>>> {
pub fn parse_assets(code: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
let cm: Lrc<SourceMap> = Default::default();
let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into());
let lexer = Lexer::new(
@@ -41,7 +41,7 @@ pub fn parse_assets(code: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>>
}
struct AssetsFinder {
assets: Vec<ParseAssetsResult<String>>,
assets: Vec<ParseAssetsResult>,
// The user will write code like:
// let sql = wmill.datatable('main')
@@ -58,7 +58,7 @@ impl Visit for AssetsFinder {
fn visit_lit(&mut self, node: &swc_ecma_ast::Lit) {
match node {
swc_ecma_ast::Lit::Str(str) => {
if let Some((kind, path)) = parse_asset_syntax(str.value.as_str()) {
if let Some((kind, path)) = parse_asset_syntax(str.value.as_str(), false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
@@ -84,19 +84,12 @@ impl Visit for AssetsFinder {
// Visit children (this may add new identifiers)
node.visit_children_with(self);
// If we find 'let sql = wmill.datatable(...)',
// but no sql`` tagged templates were used, we add
// the asset with unknown access type
for var in self.var_identifiers.keys() {
if saved_var_identifiers.contains_key(var) {
continue;
}
let (kind, ref path) = self.var_identifiers[var];
if self
.assets
.iter()
.any(|a| a.kind == kind && &a.path == path)
{
if asset_was_used(&self.assets, (kind, path)) {
continue;
}
self.assets
@@ -187,13 +180,22 @@ impl Visit for AssetsFinder {
.iter()
.map(|quasi| quasi.raw.as_str())
.collect::<Vec<_>>()
.join(" ");
.join("$1"); // placeholder for expressions
// Determine access type based on SQL keywords
let access_type = detect_sql_access_type(&sql);
let duckdb_conn_prefix = match kind {
AssetKind::DataTable => "datatable",
AssetKind::Ducklake => "ducklake",
_ => return,
};
let sql = format!("ATTACH '{duckdb_conn_prefix}://{asset_name}' AS dt; USE dt; {sql}");
self.assets
.push(ParseAssetsResult { kind, path: asset_name, access_type });
// We use the SQL parser to detect if it's a read or write query
match windmill_parser_sql::parse_assets(&sql) {
Ok(sql_assets) => {
self.assets.extend(sql_assets);
}
_ => {}
}
}
}
@@ -221,7 +223,9 @@ impl AssetsFinder {
match arg_value.map(|e| e.expr.as_ref()) {
Some(Expr::Lit(Lit::Str(Str { value, .. }))) => {
let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value);
let path = parse_asset_syntax(&value, false)
.map(|(_, p)| p)
.unwrap_or(&value);
self.assets
.push(ParseAssetsResult { kind, path: path.to_string(), access_type });
}
@@ -230,3 +234,164 @@ impl AssetsFinder {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_ts_asset_parser_load_s3() {
let input = r#"
import * as wmill from "windmill-client"
export async function main() {
wmill.loadS3File('s3:///test.csv')
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/test.csv".to_string(),
access_type: Some(R)
},])
);
}
#[test]
fn test_ts_asset_parser_unused_sql() {
let input = r#"
import * as wmill from "windmill-client"
export async function main() {
let sql = wmill.datatable('dt')
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt".to_string(),
access_type: None
},])
);
}
#[test]
fn test_ts_asset_parser_sql_read() {
let input = r#"
import * as wmill from "windmill-client"
export async function main(x: number) {
let sql = wmill.datatable('dt')
return await sql`SELECT * FROM friends WHERE age = ${x}`.fetch()
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(R)
},])
);
}
#[test]
fn test_ts_asset_parser_sql_read_write() {
let input = r#"
import * as wmill from "windmill-client"
export async function main(x: number) {
let sql = wmill.datatable('dt')
await sql`UPDATE friends SET name = 'Pierre' WHERE age = ${x}`.fetch()
let pierre = await sql`SELECT * FROM friends WHERE age = ${x}`.fetchOne()
return await sql`SELECT * FROM analytics`.fetch()
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/analytics".to_string(),
access_type: Some(R)
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "dt/friends".to_string(),
access_type: Some(RW)
},
])
);
}
#[test]
fn test_ts_asset_parser_multiple_sql_scopes() {
let input = r#"
import * as wmill from "windmill-client"
export async function main() {
async function f(x: number) {
let sql = wmill.datatable()
return await sql`SELECT * FROM friends WHERE age = ${x}`.fetch()
}
let sql = wmill.datatable('another1')
return await sql`INSERT INTO customers VALUES (${0})`.fetch()
}
function unused() {
let sql = wmill.ducklake('another2')
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1/customers".to_string(),
access_type: Some(W)
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "another2".to_string(),
access_type: None
},
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "main/friends".to_string(),
access_type: Some(R)
},
])
);
}
#[test]
fn test_ts_asset_parser_overriden_var_identifier() {
let input = r#"
import * as wmill from "windmill-client"
export async function main() {
let sql = wmill.datatable('another1')
}
function g() {
let sql = wmill.ducklake()
}
"#;
let s = parse_assets(input);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::DataTable,
path: "another1".to_string(),
access_type: None
},
ParseAssetsResult {
kind: AssetKind::Ducklake,
path: "main".to_string(),
access_type: None
},
])
);
}
}
+12 -17
View File
@@ -190,41 +190,36 @@ pub fn parse_ruby(code: &str) -> String {
#[cfg(feature = "sql-parser")]
#[wasm_bindgen]
pub fn parse_assets_sql(code: &str) -> String {
if let Ok(r) = windmill_parser_sql::parse_assets(code) {
return serde_json::to_string(&r).unwrap();
} else {
return "Invalid".to_string();
match windmill_parser_sql::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_assets_ts(code: &str) -> String {
if let Ok(r) = windmill_parser_ts::parse_assets(code) {
return serde_json::to_string(&r).unwrap();
} else {
return "Invalid".to_string();
match windmill_parser_ts::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "py-parser")]
#[wasm_bindgen]
pub fn parse_assets_py(code: &str) -> String {
if let Ok(r) = windmill_parser_py::parse_assets(code) {
return serde_json::to_string(&r).unwrap();
} else {
return "Invalid".to_string();
match windmill_parser_py::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "ansible-parser")]
#[wasm_bindgen]
pub fn parse_assets_ansible(code: &str) -> String {
let o = windmill_parser_yaml::parse_assets(code);
if let Ok(r) = o {
return serde_json::to_string(&r).unwrap();
} else {
return format!("err: {:?}", o.err().unwrap());
match windmill_parser_yaml::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
@@ -1,10 +1,10 @@
use windmill_parser::asset_parser::{
merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult,
};
merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult,
};
use crate::{parse_ansible_reqs, ResourceOrVariablePath};
pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>>> {
pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult>> {
let mut assets = vec![];
if let (_, Some(ansible_reqs), _) = parse_ansible_reqs(input)? {
if let Some(delegate_to_git_repo_details) = ansible_reqs.delegate_to_git_repo {
@@ -38,4 +38,3 @@ pub fn parse_assets(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<String>
Ok(merge_assets(assets))
}
@@ -1,6 +1,6 @@
use serde::Serialize;
#[derive(Serialize, PartialEq, Clone, Copy)]
#[derive(Serialize, PartialEq, Clone, Copy, Debug)]
#[serde(rename_all(serialize = "lowercase"))]
pub enum AssetUsageAccessType {
R,
@@ -10,7 +10,7 @@ pub enum AssetUsageAccessType {
use AssetUsageAccessType::*;
#[derive(Serialize, PartialEq, Clone, Copy)]
#[derive(Serialize, PartialEq, Clone, Copy, Debug)]
#[serde(rename_all(serialize = "lowercase"))]
pub enum AssetKind {
S3Object,
@@ -19,10 +19,10 @@ pub enum AssetKind {
DataTable,
}
#[derive(Serialize)]
pub struct ParseAssetsResult<S: AsRef<str>> {
#[derive(Serialize, Debug, PartialEq)]
pub struct ParseAssetsResult {
pub kind: AssetKind,
pub path: S,
pub path: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub access_type: Option<AssetUsageAccessType>, // None in case of ambiguity
}
@@ -34,13 +34,13 @@ pub struct DelegateToGitRepoDetails {
pub commit: Option<String>,
}
pub fn merge_assets<S: AsRef<str>>(assets: Vec<ParseAssetsResult<S>>) -> Vec<ParseAssetsResult<S>> {
let mut arr: Vec<ParseAssetsResult<S>> = vec![];
pub fn merge_assets(assets: Vec<ParseAssetsResult>) -> Vec<ParseAssetsResult> {
let mut arr: Vec<ParseAssetsResult> = vec![];
for asset in assets {
// Remove duplicates
if let Some(existing) = arr
.iter_mut()
.find(|x| x.path.as_ref() == asset.path.as_ref() && x.kind == asset.kind)
.find(|x| x.path == asset.path && x.kind == asset.kind)
{
// merge access types
existing.access_type = match (asset.access_type, existing.access_type) {
@@ -54,12 +54,33 @@ pub fn merge_assets<S: AsRef<str>>(assets: Vec<ParseAssetsResult<S>>) -> Vec<Par
arr.push(asset);
}
}
arr.sort_by(|a, b| a.path.as_ref().cmp(b.path.as_ref()));
arr.sort_by(|a, b| a.path.cmp(&b.path));
arr
}
pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
if s.starts_with("s3://") {
// Will return false if the user assigned an asset to a variable like:
// let sql = wmill.datatable('main')
// But never used it. In that case we don't know which table is being used,
// but we still want to add the main datatable as an asset with unknown access type.
//
// This function takes care of the fact that assets can be suffixed (e.g. "main/users")
pub fn asset_was_used(assets: &Vec<ParseAssetsResult>, (kind, path): (AssetKind, &String)) -> bool {
assets.iter().any(|a| {
let a_path = a.path.as_str();
let has_same_path_base = a_path
.strip_prefix(path)
.map(|p| p.starts_with('/'))
.unwrap_or(false);
(has_same_path_base || a_path == path) && a.kind == kind
})
}
pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(AssetKind, &str)> {
if enable_default_syntax && s == "datatable" {
Some((AssetKind::DataTable, "main"))
} else if enable_default_syntax && s == "ducklake" {
Some((AssetKind::Ducklake, "main"))
} else if s.starts_with("s3://") {
Some((AssetKind::S3Object, &s[5..]))
} else if s.starts_with("res://") {
Some((AssetKind::Resource, &s[6..]))
@@ -68,41 +89,8 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
} else if s.starts_with("ducklake://") {
Some((AssetKind::Ducklake, &s[11..]))
} else if s.starts_with("datatable://") {
Some((AssetKind::DataTable, &s[11..]))
Some((AssetKind::DataTable, &s[12..]))
} else {
None
}
}
pub fn detect_sql_access_type(sql: &str) -> Option<AssetUsageAccessType> {
let first_kw = sql
.trim()
.split_whitespace()
.next()
.unwrap_or("")
.to_lowercase();
// Check for write operations
let has_write = first_kw.starts_with("insert")
|| first_kw.starts_with("update")
|| first_kw.starts_with("delete")
|| first_kw.starts_with("drop")
|| first_kw.starts_with("create")
|| first_kw.starts_with("alter")
|| first_kw.starts_with("truncate")
|| first_kw.starts_with("merge");
// Check for read operations
let has_read = first_kw.starts_with("select")
|| first_kw.starts_with("with") // CTEs, usually for reads
|| first_kw.starts_with("show")
|| first_kw.starts_with("describe")
|| first_kw.starts_with("explain");
match (has_read, has_write) {
(true, true) => Some(RW),
(true, false) => Some(R),
(false, true) => Some(W),
(false, false) => None, // Unknown - couldn't determine
}
}
+3 -1
View File
@@ -39,7 +39,9 @@ async fn list_assets(
)
)) as "list!: _"
FROM asset
LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1
LEFT JOIN resource ON asset.kind = 'resource'
AND array_to_string((string_to_array(asset.path, '/'))[1:3], '/') = resource.path -- With specific table, asset path can be e.g u/diego/pg_db/table_name
AND resource.workspace_id = $1
WHERE asset.workspace_id = $1
AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)
AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))
+55 -112
View File
@@ -79,11 +79,11 @@
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.574.1",
"windmill-parser-wasm-py": "1.589.3",
"windmill-parser-wasm-regex": "1.589.3",
"windmill-parser-wasm-py": "1.590.0",
"windmill-parser-wasm-regex": "1.590.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.558.1",
"windmill-parser-wasm-ts": "1.589.3",
"windmill-parser-wasm-ts": "1.590.0",
"windmill-parser-wasm-yaml": "1.561.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.1",
@@ -263,7 +263,6 @@
"integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/helper-validator-identifier": "^7.27.1",
"js-tokens": "^4.0.0",
@@ -279,7 +278,6 @@
"integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=6.9.0"
}
@@ -1861,7 +1859,6 @@
}
],
"license": "MIT",
"peer": true,
"engines": {
"node": "^14 || ^16 || >=18"
},
@@ -2371,6 +2368,7 @@
"integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@floating-ui/core": "^1.3.1",
"@floating-ui/dom": "^1.4.5",
@@ -2569,6 +2567,7 @@
"resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz",
"integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==",
"license": "MIT",
"peer": true,
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/popperjs"
@@ -2914,6 +2913,7 @@
"integrity": "sha512-7TSvMrCdmig5TMyYDW876C5FljhA0wlGixtvASCiqUqtLfmyEEpaysXjC7GhR5mWcGRrCGF+L2Bl1eEaW1wTCA==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@standard-schema/spec": "^1.0.0",
"@sveltejs/acorn-typescript": "^1.0.5",
@@ -2991,6 +2991,7 @@
"integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@sveltejs/vite-plugin-svelte-inspector": "^5.0.0",
"debug": "^4.4.1",
@@ -3497,8 +3498,7 @@
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/@types/ms": {
"version": "2.1.0",
@@ -3511,8 +3511,7 @@
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz",
"integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/@types/semver": {
"version": "7.7.1",
@@ -3582,6 +3581,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -3772,6 +3772,7 @@
"integrity": "sha512-94yVpDbb+ykiT7mK6ToonGnq2GIHEQGBTZTAzGxBGQXcVNCh54YKC2/WkfaDzxy0m6Kgw05kq3FYHKHu+wRdIA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@vitest/browser": "4.0.15",
"@vitest/mocker": "4.0.15",
@@ -4021,6 +4022,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -4074,6 +4076,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
"integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.3",
"fast-uri": "^3.0.1",
@@ -4219,7 +4222,6 @@
"integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -4247,7 +4249,6 @@
"integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -4340,8 +4341,7 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz",
"integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/base64-js": {
"version": "1.5.1",
@@ -4468,6 +4468,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"baseline-browser-mapping": "^2.8.9",
"caniuse-lite": "^1.0.30001746",
@@ -4665,7 +4666,6 @@
"integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"camelcase": "^6.3.0",
"map-obj": "^4.1.0",
@@ -4685,7 +4685,6 @@
"integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -4699,7 +4698,6 @@
"integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -4713,7 +4711,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -4822,6 +4819,7 @@
"resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz",
"integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==",
"license": "MIT",
"peer": true,
"dependencies": {
"@kurkle/color": "^0.3.0"
},
@@ -5062,7 +5060,6 @@
"integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"import-fresh": "^3.3.0",
"js-yaml": "^4.1.0",
@@ -5127,7 +5124,6 @@
"integrity": "sha512-IQOkD3hbR5KrN93MtcYuad6YPuTSUhntLHDuLEbFWE+ff2/XSZNdZG+LcbbIW5AXKg/WFIfYItIzVoHngHXZzA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12 || >=16"
}
@@ -5393,6 +5389,7 @@
"resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
"integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
"license": "ISC",
"peer": true,
"engines": {
"node": ">=12"
}
@@ -5446,6 +5443,7 @@
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
"integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/runtime": "^7.21.0"
},
@@ -5486,7 +5484,6 @@
"integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -5500,7 +5497,6 @@
"integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"decamelize": "^1.1.0",
"map-obj": "^1.0.0"
@@ -5518,7 +5514,6 @@
"integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -5529,7 +5524,6 @@
"integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -6015,7 +6009,6 @@
"integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"is-arrayish": "^0.2.1"
}
@@ -6103,6 +6096,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -6649,7 +6643,6 @@
"integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 4.9.1"
}
@@ -7135,7 +7128,6 @@
"integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"global-prefix": "^3.0.0"
},
@@ -7149,7 +7141,6 @@
"integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ini": "^1.3.5",
"kind-of": "^6.0.2",
@@ -7165,7 +7156,6 @@
"integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"isexe": "^2.0.0"
},
@@ -7215,8 +7205,7 @@
"resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz",
"integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/gopd": {
"version": "1.2.0",
@@ -7252,6 +7241,7 @@
"resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz",
"integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==",
"license": "MIT",
"peer": true,
"engines": {
"node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
}
@@ -7310,7 +7300,6 @@
"integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=6"
}
@@ -7532,7 +7521,6 @@
"integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"lru-cache": "^6.0.0"
},
@@ -7546,7 +7534,6 @@
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"yallist": "^4.0.0"
},
@@ -7559,8 +7546,7 @@
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true,
"license": "ISC",
"peer": true
"license": "ISC"
},
"node_modules/html-tags": {
"version": "3.3.1",
@@ -7568,7 +7554,6 @@
"integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
},
@@ -7652,7 +7637,6 @@
"integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -7673,7 +7657,6 @@
"integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -7745,8 +7728,7 @@
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
"integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/is-binary-path": {
"version": "2.1.0",
@@ -7851,7 +7833,6 @@
"integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -7862,7 +7843,6 @@
"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -7961,8 +7941,7 @@
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/js-yaml": {
"version": "4.1.0",
@@ -7998,8 +7977,7 @@
"resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
"integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/json-refs": {
"version": "3.0.15",
@@ -8112,7 +8090,6 @@
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=0.10.0"
}
@@ -8691,8 +8668,7 @@
"resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
"integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/lodash.uniq": {
"version": "4.5.0",
@@ -8770,7 +8746,6 @@
"integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
},
@@ -8821,7 +8796,6 @@
"integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==",
"dev": true,
"license": "MIT",
"peer": true,
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
@@ -9062,7 +9036,6 @@
"integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/minimist": "^1.2.2",
"camelcase-keys": "^7.0.0",
@@ -9090,7 +9063,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -9784,7 +9756,6 @@
"integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"arrify": "^1.0.1",
"is-plain-obj": "^1.1.0",
@@ -9863,6 +9834,7 @@
"resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-21.6.0.tgz",
"integrity": "sha512-YTxKRHe9d4TvyEzWIqLpJXLyZyO4xFlLgrkgHoWBpomm6gIuwaRJJRpapBZf24oG8AhkniZSPg2iv/84M+ho6g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "21.6.0",
"@codingame/monaco-vscode-api": "21.6.0"
@@ -10125,7 +10097,6 @@
"integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"hosted-git-info": "^4.0.1",
"is-core-module": "^2.5.0",
@@ -10477,7 +10448,6 @@
"integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@babel/code-frame": "^7.0.0",
"error-ex": "^1.3.1",
@@ -10823,6 +10793,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"nanoid": "^3.3.11",
"picocolors": "^1.1.1",
@@ -11011,6 +10982,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"lilconfig": "^3.0.0",
"yaml": "^2.3.4"
@@ -11400,8 +11372,7 @@
"resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz",
"integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/postcss-safe-parser": {
"version": "6.0.0",
@@ -11576,6 +11547,7 @@
"resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz",
"integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==",
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin/prettier.cjs"
},
@@ -11880,7 +11852,6 @@
"integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/normalize-package-data": "^2.4.0",
"normalize-package-data": "^3.0.2",
@@ -11900,7 +11871,6 @@
"integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"find-up": "^5.0.0",
"read-pkg": "^6.0.0",
@@ -11919,7 +11889,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -11933,7 +11902,6 @@
"integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==",
"dev": true,
"license": "(MIT OR CC0-1.0)",
"peer": true,
"engines": {
"node": ">=10"
},
@@ -11976,7 +11944,6 @@
"integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"indent-string": "^5.0.0",
"strip-indent": "^4.0.0"
@@ -12572,7 +12539,6 @@
"integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"ansi-styles": "^4.0.0",
"astral-regex": "^2.0.0",
@@ -12649,7 +12615,6 @@
"integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"spdx-expression-parse": "^3.0.0",
"spdx-license-ids": "^3.0.0"
@@ -12660,8 +12625,7 @@
"resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz",
"integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==",
"dev": true,
"license": "CC-BY-3.0",
"peer": true
"license": "CC-BY-3.0"
},
"node_modules/spdx-expression-parse": {
"version": "3.0.1",
@@ -12669,7 +12633,6 @@
"integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"spdx-exceptions": "^2.1.0",
"spdx-license-ids": "^3.0.0"
@@ -12680,8 +12643,7 @@
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz",
"integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==",
"dev": true,
"license": "CC0-1.0",
"peer": true
"license": "CC0-1.0"
},
"node_modules/sprintf-js": {
"version": "1.0.3",
@@ -12775,7 +12737,6 @@
"integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -12801,8 +12762,7 @@
"resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz",
"integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==",
"dev": true,
"license": "ISC",
"peer": true
"license": "ISC"
},
"node_modules/style-to-object": {
"version": "0.4.4",
@@ -12851,7 +12811,6 @@
"integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@csstools/css-parser-algorithms": "^2.3.1",
"@csstools/css-tokenizer": "^2.2.0",
@@ -12934,7 +12893,6 @@
}
],
"license": "MIT-0",
"peer": true,
"engines": {
"node": "^14 || ^16 || >=18"
},
@@ -12948,7 +12906,6 @@
"integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"flat-cache": "^3.2.0"
},
@@ -12961,8 +12918,7 @@
"resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz",
"integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==",
"dev": true,
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/stylelint/node_modules/postcss-selector-parser": {
"version": "6.1.2",
@@ -12985,7 +12941,6 @@
"integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=8"
}
@@ -13120,7 +13075,6 @@
"integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"has-flag": "^4.0.0",
"supports-color": "^7.0.0"
@@ -13150,6 +13104,7 @@
"resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz",
"integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@jridgewell/remapping": "^2.3.4",
"@jridgewell/sourcemap-codec": "^1.5.0",
@@ -13245,21 +13200,6 @@
}
}
},
"node_modules/svelte-check/node_modules/picomatch": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"dev": true,
"license": "MIT",
"optional": true,
"peer": true,
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/jonschlinkert"
}
},
"node_modules/svelte-eslint-parser": {
"version": "0.43.0",
"resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz",
@@ -13462,8 +13402,7 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz",
"integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==",
"dev": true,
"peer": true
"dev": true
},
"node_modules/svgo": {
"version": "3.3.2",
@@ -13514,7 +13453,6 @@
"integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==",
"dev": true,
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"ajv": "^8.0.1",
"lodash.truncate": "^4.4.2",
@@ -13542,6 +13480,7 @@
"integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@alloc/quick-lru": "^5.2.0",
"arg": "^5.0.2",
@@ -13784,6 +13723,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -13846,7 +13786,6 @@
"integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==",
"dev": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -13945,6 +13884,7 @@
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -14159,7 +14099,6 @@
"integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==",
"dev": true,
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"spdx-correct": "^3.0.0",
"spdx-expression-parse": "^3.0.0"
@@ -14214,6 +14153,7 @@
"integrity": "sha512-8wKihlF6EDF8grimwd7GPOhLkQkSIgj6Hlcp0CXhtO3HAXeUUqhgZmJmn07OF8e4PbTusMX6Yxmy1BptVRZsdw==",
"devOptional": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@oxc-project/runtime": "0.99.0",
"fdir": "^6.5.0",
@@ -14326,6 +14266,7 @@
"integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
"devOptional": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=12"
},
@@ -14359,6 +14300,7 @@
"integrity": "sha512-n1RxDp8UJm6N0IbJLQo+yzLZ2sQCDyl1o0LeugbPWf8+8Fttp29GghsQBjYJVmWq3gBFfe9Hs1spR44vovn2wA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@vitest/expect": "4.0.15",
"@vitest/mocker": "4.0.15",
@@ -14691,14 +14633,14 @@
"integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA=="
},
"node_modules/windmill-parser-wasm-py": {
"version": "1.589.3",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.589.3.tgz",
"integrity": "sha512-ob1auYfqDyjaliXVluSKAuhfW09TefJ1rJjfDBnV5KCumkTBYGSchrGU+2AJYiuwCtljRE5CzTPBAo6JTaSKHA=="
"version": "1.590.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.590.0.tgz",
"integrity": "sha512-8TojFdxzoRYsx+UqYuhwE/rBZqH9OBs8FWe7A+GLaA3HtKe4NDb57mKpiXFPDnEYXaSF8gs8g+IvNrZJq4qhig=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.589.3",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.589.3.tgz",
"integrity": "sha512-yfYxHtrTuW+vqovPvu1h0t9XmC1cLzE6Quxb6fke3rxKYxQ3scO31zvIeZs2hGlvi/nzJjtCCgmZRyqD6RVQXQ=="
"version": "1.590.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.590.0.tgz",
"integrity": "sha512-M+KqE6x66Utjp/V8XbHHXHtjzXYpkLcd3jINi1hohhYhNru950c68psyu8X7WmXZBnv+AL0NvcOsrBDHW5q0iQ=="
},
"node_modules/windmill-parser-wasm-ruby": {
"version": "1.526.1",
@@ -14711,9 +14653,9 @@
"integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.589.3",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.589.3.tgz",
"integrity": "sha512-PNyzEWo3qJxlocz1qmeh4IOxm8TzCeyAqqXFO4oyz9Lu6WAyCaSfefhZfaZG01wz3J1hJTZxymB9dDiv0VxgzQ=="
"version": "1.590.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.590.0.tgz",
"integrity": "sha512-OHeIkuYd7RiC946fSzb9HnHSGJzcFbtz57LfRCnRuF/ky7rq26Csr+4uArpTmtBxiyHMVR3U22+qiNe2TjYxJQ=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.561.0",
@@ -14864,7 +14806,6 @@
"integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==",
"dev": true,
"license": "ISC",
"peer": true,
"dependencies": {
"imurmurhash": "^0.1.4",
"signal-exit": "^4.0.1"
@@ -14879,6 +14820,7 @@
"integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==",
"devOptional": true,
"license": "MIT",
"peer": true,
"engines": {
"node": ">=10.0.0"
},
@@ -15082,7 +15024,6 @@
"integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==",
"dev": true,
"license": "ISC",
"peer": true,
"engines": {
"node": ">=10"
}
@@ -15102,6 +15043,7 @@
"resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz",
"integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==",
"license": "MIT",
"peer": true,
"dependencies": {
"lib0": "^0.2.99"
},
@@ -15144,6 +15086,7 @@
"resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz",
"integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
+3 -3
View File
@@ -153,11 +153,11 @@
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.574.1",
"windmill-parser-wasm-py": "1.589.3",
"windmill-parser-wasm-regex": "1.589.3",
"windmill-parser-wasm-py": "1.590.0",
"windmill-parser-wasm-regex": "1.590.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.558.1",
"windmill-parser-wasm-ts": "1.589.3",
"windmill-parser-wasm-ts": "1.590.0",
"windmill-parser-wasm-yaml": "1.561.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.1",
+4
View File
@@ -208,4 +208,8 @@ svelte-virtual-list-contents > * + * {
input[type="time"]::-webkit-calendar-picker-indicator {
margin: 0;
padding: 0;
}
.svelte-flow__edges {
z-index: -10;
}
+11 -6
View File
@@ -22,6 +22,7 @@
dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps
dbSchemaOps: IDbSchemaOps
refresh?: () => void
initialTableKey?: string
}
let {
dbType,
@@ -30,7 +31,8 @@
dbSchemaOps,
getColDefs,
dbSupportsSchemas,
refresh
refresh,
initialTableKey
}: Props = $props()
let schemaKeys = $derived(Object.keys(dbSchema.schema ?? {}))
@@ -42,10 +44,13 @@
$effect(() => {
if (!selected.schemaKey && schemaKeys.length) {
selected = {
schemaKey:
'public' in dbSchema.schema ? 'public' : 'dbo' in dbSchema.schema ? 'dbo' : schemaKeys[0]
}
let schemaKey =
'public' in dbSchema.schema ? 'public' : 'dbo' in dbSchema.schema ? 'dbo' : schemaKeys[0]
let tableKey =
initialTableKey && dbSchema.schema?.[schemaKey]?.[initialTableKey]
? initialTableKey
: undefined
selected = { schemaKey, tableKey }
}
})
@@ -122,7 +127,7 @@
onConfirm: async () => {
askingForConfirmation && (askingForConfirmation.loading = true)
try {
await dbSchemaOps.onDelete({ tableKey })
await dbSchemaOps.onDelete({ tableKey, schema: selected.schemaKey })
refresh?.()
sendUserToast(`Table '${tableKey}' deleted successfully`)
} catch (e) {
@@ -194,6 +194,7 @@
input: _input,
workspace: $workspaceStore
})}
initialTableKey={input.specificTable}
{dbType}
{refresh}
/>
+3 -3
View File
@@ -741,7 +741,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
}
}}
tooltip="Attach a Ducklake to your scripts. Ducklake allows you to manipulate large data on S3 blob files through a traditional SQL interface."
documentationLink="https://www.windmill.dev/docs/core_concepts/ducklake"
documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake"
itemName="ducklake"
loadItems={async () =>
(await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? 'NO_W' })).map(
@@ -783,7 +783,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
}
}}
tooltip="Attach a datatable to your script."
documentationLink="https://www.windmill.dev/docs/core_concepts/data_tables"
documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables"
itemName="data table"
loadItems={async () =>
(await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map(
@@ -836,7 +836,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
onSelectAndClose={(s3obj) => {
let s = `'${formatS3Object(s3obj)}'`
if (lang === 'duckdb') {
editor?.insertAtCursor(`SELECT * FROM ${s}`)
editor?.insertAtCursor(`SELECT * FROM ${s};`)
} else if (lang === 'python3') {
if (!editor?.getCode().includes('import wmill')) {
editor?.insertAtBeginning('import wmill\n')
@@ -58,20 +58,28 @@
{btnClasses}
on:click={async () => {
if (asset.kind === 'resource' && isDbType(_resourceMetadata?.resource_type)) {
let resourcePath = asset.path.split('/').slice(0, 3).join('/')
let specificTable = asset.path.split('/')[3] as string | undefined
dbManagerDrawer?.openDrawer({
type: 'database',
resourceType: _resourceMetadata.resource_type,
resourcePath: asset.path
resourcePath,
specificTable
})
} else if (asset.kind === 's3object' && isS3Uri(assetUri)) {
s3FilePicker?.open(assetUri)
} else if (asset.kind === 'ducklake') {
dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake: asset.path })
let ducklake = asset.path.split('/')[0]
let specificTable = asset.path.split('/')[1] as string | undefined
dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificTable })
} else if (asset.kind === 'datatable') {
let datatable = asset.path.split('/')[0]
let specificTable = asset.path.split('/')[1] as string | undefined
dbManagerDrawer?.openDrawer({
type: 'database',
resourceType: 'postgresql',
resourcePath: `datatable://${asset.path}`
resourcePath: `datatable://${datatable}`,
specificTable
})
}
onClick?.()
@@ -58,6 +58,7 @@
import { copilotInfo } from '$lib/aiStore'
import JsonInputs from '$lib/components/JsonInputs.svelte'
import Toggle from './Toggle.svelte'
import { deepEqual } from 'fast-equals'
interface Props {
// Exported
@@ -158,12 +159,14 @@
$effect(() => {
;[lang, code]
untrack(() => {
inferAssets(lang, code).then((newAssets: AssetWithAltAccessType[]) => {
inferAssets(lang, code).then((inferAssetsResult) => {
if (inferAssetsResult.status === 'error') return
let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[]
for (const asset of newAssets) {
const old = assets?.find((a) => assetEq(a, asset))
if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
}
assets = newAssets
if (!deepEqual(assets, newAssets)) assets = newAssets
})
if (lang === 'ansible') {
@@ -1,6 +1,11 @@
import type { DbType } from '$lib/components/dbTypes'
export function makeDeleteTableQuery(tableKey: string, resourceType: DbType): string {
export function makeDeleteTableQuery(
tableKey: string,
resourceType: DbType,
schema?: string
): string {
if (schema?.length) tableKey = `${schema}.${tableKey}`
// same for all sql dbs
return `DROP TABLE ${tableKey};`
}
@@ -28,10 +28,15 @@
datatableNotFound = false,
onClick
}: Props = $props()
let resourceDataCacheValue = $derived.by(() => {
let truncatedPath = asset.path.split('/').slice(0, 3).join('/')
return resourceDataCache[truncatedPath]
})
</script>
<div class="flex gap-2 items-center">
{#if asset.kind === 'resource' && resourceDataCache[asset.path] !== undefined}
{#if asset.kind === 'resource' && resourceDataCacheValue !== undefined}
<Button
startIcon={{ icon: Edit2 }}
variant="default"
@@ -40,7 +45,7 @@
on:click={() => (resourceEditorDrawer?.initEdit(asset.path), onClick?.())}
/>
{/if}
{#if (asset.kind === 'resource' && resourceDataCache[asset.path] === undefined) || ducklakeNotFound || datatableNotFound}
{#if (asset.kind === 'resource' && resourceDataCacheValue === undefined) || ducklakeNotFound || datatableNotFound}
<Popover contentClasses="px-3 py-2">
<svelte:fragment slot="trigger">
<Button
@@ -61,19 +66,19 @@
<Button wrapperClasses="mt-1" href="/workspace_settings?tab=windmill_data_tables">
Go to Data Table settings
</Button>
{:else if asset.kind === 'resource' && resourceDataCache[asset.path] === undefined}
{:else if asset.kind === 'resource' && resourceDataCacheValue === undefined}
<Button wrapperClasses="mt-1" href="/resources">Go to Resources</Button>
{/if}
</svelte:fragment>
</Popover>
{:else if assetCanBeExplored(asset, { resource_type: resourceDataCache[asset.path] })}
{:else if assetCanBeExplored(asset, { resource_type: resourceDataCacheValue })}
<ExploreAssetButton
{asset}
{s3FilePicker}
{dbManagerDrawer}
onClick={() => onClick?.()}
noText
_resourceMetadata={{ resource_type: resourceDataCache[asset.path] }}
_resourceMetadata={{ resource_type: resourceDataCacheValue }}
/>
{/if}
</div>
@@ -84,10 +84,14 @@
}
for (const asset of assets) {
if (asset.kind !== 'resource' || asset.path in resourceDataCache) continue
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! })
.then((resource) => (resourceDataCache[asset.path] = resource.resource_type))
.catch((err) => (resourceDataCache[asset.path] = undefined))
if (asset.kind == 'resource') {
let truncatedPath = asset.path.split('/').slice(0, 3).join('/')
if (truncatedPath in resourceDataCache) continue
resourceDataCache[truncatedPath] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! })
.then((r) => (resourceDataCache[truncatedPath] = r.resource_type))
.catch((err) => console.error("Couldn't fetch resource", truncatedPath, err))
}
}
})
})
@@ -107,7 +111,7 @@
'text-xs flex items-center gap-1.5 px-2 rounded-md relative',
'border',
'bg-surface hover:bg-surface-hover active:bg-surface',
'transition-all hover:text-primary cursor-pointer'
'transition-all hover:text-primary backdrop-blur-md cursor-pointer'
)}
>
<div
@@ -128,11 +132,11 @@
{@const ducklakeNotFound =
asset.kind === 'ducklake' &&
ducklakes.current &&
!ducklakes.current.find((name) => name === asset.path)}
!ducklakes.current.find((name) => name === asset.path.split('/')[0])}
{@const datatableNotFound =
asset.kind === 'datatable' &&
datatables.current &&
!datatables.current.find((name) => name === asset.path)}
!datatables.current.find((name) => name === asset.path.split('/')[0])}
<li
class="text-sm px-3 h-12 flex gap-3 items-center"
onmouseenter={() => onHoverLi?.(asset, 'enter')}
@@ -146,9 +150,7 @@
<div
class={twMerge(
'text-xs font-normal border text-primary w-10 p-1 text-center rounded-md',
!asset.access_type && !asset.alt_access_type
? 'text-orange-500 !border-orange-500'
: '',
!asset.access_type ? 'text-orange-600 !border-orange-600' : '',
!asset.access_type ? 'hover:bg-surface active:opacity-80' : ''
)}
>
@@ -162,7 +164,11 @@
<span class="text-xs">Please select manually </span>
</span>
<div class="flex items-center gap-2">
<ToggleButtonGroup bind:selected={asset.alt_access_type} class="max-w-fit">
<ToggleButtonGroup
allowEmpty
bind:selected={asset.alt_access_type}
class="max-w-fit"
>
{#snippet children({ item })}
<ToggleButton value="r" label="Read" {item} />
<ToggleButton value="w" label="Write" {item} />
@@ -34,11 +34,11 @@
(x) => x.kind + x.path
)
}
if (job.job_kind === 'script') {
return [
...(await inferAssets(job.language!, job.raw_code ?? '')),
...parseInputArgsAssets(job.args ?? {})
]
let inferAssetsResult = await inferAssets(job.language!, job.raw_code ?? '')
let assets = inferAssetsResult.status === 'ok' ? inferAssetsResult.assets : []
return [...assets, ...parseInputArgsAssets(job.args ?? {})]
}
return []
}
@@ -53,10 +53,14 @@
let resourceDataCache: Record<string, string | undefined> = $state({})
$effect(() => {
for (const asset of assets.value ?? []) {
if (asset.kind !== 'resource' || asset.path in resourceDataCache) continue
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! })
.then((resource) => (resourceDataCache[asset.path] = resource.resource_type))
.catch((err) => (resourceDataCache[asset.path] = undefined))
if (asset.kind == 'resource') {
let truncatedPath = asset.path.split('/').slice(0, 3).join('/')
if (truncatedPath in resourceDataCache) continue
resourceDataCache[truncatedPath] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! })
.then((r) => (resourceDataCache[truncatedPath] = r.resource_type))
.catch((err) => console.error("Couldn't fetch resource", truncatedPath, err))
}
}
})
+5 -1
View File
@@ -28,6 +28,10 @@ export function formatAsset(asset: Asset): string {
return 'unknown'
}
export function formatShortAssetPath(asset: Asset): string {
return asset.path.split('/').pop() || asset.path
}
export function getAssetUsagePageUri(usage: ListAssetsResponse[number]['usages'][number]) {
if (usage.kind === 'script') {
return `/scripts/get/${usage.path}`
@@ -93,8 +97,8 @@ export function formatAssetAccessType(accessType: AssetUsageAccessType | undefin
}
export function getAccessType(asset: AssetWithAltAccessType): AssetUsageAccessType | undefined {
if (asset.alt_access_type) return asset.alt_access_type
if (asset.access_type) return asset.access_type
if (asset.alt_access_type) return asset.alt_access_type
}
export function getFlowModuleAssets(
+3 -3
View File
@@ -112,7 +112,7 @@ export function dbTableOpsWithPreviewScripts({
}
export type IDbSchemaOps = {
onDelete: (params: { tableKey: string }) => Promise<void>
onDelete: (params: { tableKey: string; schema?: string }) => Promise<void>
onCreate: (params: { values: CreateTableValues; schema?: string }) => Promise<void>
previewCreateSql: (params: { values: CreateTableValues; schema?: string }) => string
}
@@ -128,8 +128,8 @@ export function dbSchemaOpsWithPreviewScripts({
const dbArg = getDatabaseArg(input)
const language = getLanguageByResourceType(dbType)
return {
onDelete: async ({ tableKey }) => {
let deleteQuery = makeDeleteTableQuery(tableKey, dbType)
onDelete: async ({ tableKey, schema }) => {
let deleteQuery = makeDeleteTableQuery(tableKey, dbType, schema)
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
await runScriptAndPollResult({
workspace,
+6 -1
View File
@@ -3,8 +3,13 @@ export type DbInput =
type: 'database'
resourceType: DbType
resourcePath: string
specificTable?: string
}
| {
type: 'ducklake'
ducklake: string
specificTable?: string
}
| { type: 'ducklake'; ducklake: string }
export type DbType = (typeof dbTypes)[number]
export const dbTypes = [
@@ -73,13 +73,14 @@
(m) => getFlowModuleAssets(m, flowGraphAssetsCtx?.val.additionalAssetsMap) ?? []
) ?? []
for (const asset of assets) {
if (asset.kind !== 'resource' || asset.path in resMetadataCache) continue
resMetadataCache[asset.path] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! })
.then((r) => (resMetadataCache[asset.path] = { resource_type: r.resource_type }))
.catch((err) => {
console.error("Couldn't fetch resource", asset.path, err)
})
if (asset.kind == 'resource') {
let truncatedPath = asset.path.split('/').slice(0, 3).join('/')
if (truncatedPath in resMetadataCache) continue
resMetadataCache[truncatedPath] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! })
.then((r) => (resMetadataCache[truncatedPath] = { resource_type: r.resource_type }))
.catch((err) => console.error("Couldn't fetch resource", truncatedPath, err))
}
}
})
@@ -119,14 +120,14 @@
})
async function parseAndUpdateRawScriptModule(v: RawScript) {
try {
let parsedAssets: AssetWithAltAccessType[] = await inferAssets(v.language, v.content)
for (const asset of parsedAssets) {
const old = v.assets?.find((a) => assetEq(a, asset))
if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
}
if (!deepEqual(v.assets, parsedAssets)) v.assets = parsedAssets
} catch (e) {}
let inferAssetsResult = await inferAssets(v.language, v.content)
if (inferAssetsResult.status === 'error') return
let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[]
for (const asset of newAssets) {
const old = v.assets?.find((a) => assetEq(a, asset))
if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
}
if (!deepEqual(v.assets, newAssets)) v.assets = newAssets
}
// Check for raw script modules whose assets were not parsed. Useful for flows created
@@ -122,7 +122,7 @@
}}
>
<!-- Invisible hover area to maintain consistent height -->
<div class="absolute w-full h-[18px]"></div>
<div class="absolute w-full h-[20px]"></div>
<div
class={twMerge(
'absolute w-full top-1',
@@ -43,5 +43,6 @@
<BaseEdge
path={edgePath}
{markerEnd}
class={twMerge($useDataflow ? 'hidden' : '', data.class ?? '')}
class={twMerge($useDataflow ? 'hidden' : '', 'pointer-events-none', data.class ?? '')}
interactionWidth={0}
/>
@@ -220,6 +220,7 @@
import {
assetEq,
formatAssetKind,
formatShortAssetPath,
getAccessType,
type AssetWithAltAccessType
} from '$lib/components/assets/lib'
@@ -246,9 +247,11 @@
let { data }: Props = $props()
const isSelected = $derived(assetEq(flowGraphAssetsCtx?.val.selectedAsset, data.asset))
const cachedResourceMetadata = $derived(
flowGraphAssetsCtx?.val.resourceMetadataCache[data.asset.path]
)
const cachedResourceMetadata = $derived.by(() => {
if (data.asset.kind !== 'resource') return undefined
let truncatedPath = data.asset.path.split('/').slice(0, 3).join('/')
return flowGraphAssetsCtx?.val.resourceMetadataCache[truncatedPath]
})
const usageCount = $derived(flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset))
const colors = $derived(getNodeColorClasses(undefined, isSelected))
</script>
@@ -271,14 +274,11 @@
>
<AssetGenericIcon
assetKind={data.asset.kind}
fill={''}
class="shrink-0 ml-1 {isSelected
? 'fill-accent stroke-accent'
: 'fill-tertiary stroke-tertiary'}"
class="shrink-0 ml-1 {isSelected ? 'text-accent' : 'text-tertiary'}"
size="16px"
/>
<span class="text-3xs truncate flex-1">
{data.asset.path}
{formatShortAssetPath(data.asset)}
</span>
{#if data.asset.kind === 'resource' && cachedResourceMetadata === undefined}
<Tooltip class={'pr-1 flex items-center justify-center'}>
@@ -9,6 +9,7 @@
import type { FlowGraphAssetContext } from '$lib/components/flows/types'
import { getContext } from 'svelte'
import { assetEq } from '$lib/components/assets/lib'
import { getNodeColorClasses } from '../../util'
interface Props {
data: AssetsOverflowedN['data']
@@ -33,6 +34,7 @@
wasOpenedBecauseOfExternalSelected = false
}
})
const colors = $derived(getNodeColorClasses(undefined, includesSelected))
</script>
<NodeWrapper>
@@ -43,9 +45,11 @@
usePointerDownOutside
bind:isOpen
class={twMerge(
'!w-full text-2xs font-normal bg-surface h-6 pr-0.5 flex justify-center items-center rounded-sm text-primary border',
includesSelected ? 'bg-surface-secondary border-surface-inverse' : 'border-transparent',
'hover:bg-surface-secondary hover:border-surface-inverse active:opacity-55'
'!w-full text-2xs font-normal h-6 pr-0.5 flex justify-center items-center rounded-md text-primary drop-shadow-base',
'hover:bg-surface-hover active:bg-surface active:opacity-80',
colors.bg,
colors.outline,
colors.text
)}
placement="top"
>
@@ -1,29 +1,25 @@
<script lang="ts">
import { Pyramid } from 'lucide-svelte'
import { BoxesIcon, DatabaseIcon, FileIcon, Pyramid } from 'lucide-svelte'
import type { AssetKind } from '../assets/lib'
import AssetResIcon from './AssetResIcon.svelte'
import AssetS3Icon from './AssetS3Icon.svelte'
import AssetDucklakeIcon from './AssetDucklakeIcon.svelte'
import AssetDatabaseIcon from './AssetDatabaseIcon.svelte'
import DucklakeIcon from './DucklakeIcon.svelte'
interface Props {
size?: string
fill?: string
assetKind: AssetKind
class?: string
}
let { assetKind, fill, size, class: className = '' }: Props = $props()
let { assetKind, size, class: className = '' }: Props = $props()
</script>
{#if assetKind == 's3object'}
<AssetS3Icon {fill} width={size} height={size} class={className} />
{:else if assetKind == 'resource'}
<AssetResIcon {fill} width={size} height={size} class={className} />
<FileIcon {size} class={className} />
{:else if assetKind == 'datatable'}
<AssetDatabaseIcon {fill} width={size} height={size} class={className} />
<DatabaseIcon {size} class={className} />
{:else if assetKind == 'ducklake'}
<AssetDucklakeIcon {fill} width={size} height={size} class={className} />
<DucklakeIcon {size} class={className} />
{:else if assetKind == 'resource'}
<BoxesIcon {size} class={className} />
{:else}
<Pyramid {size} color={fill} class={'!fill-none ' + className} />
<Pyramid {size} class={className} />
{/if}
@@ -1,14 +1,16 @@
<script lang="ts">
interface Props {
size?: number
size?: number | string
class?: string
}
let { size = 24 }: Props = $props()
let { size = 24, class: className }: Props = $props()
</script>
<svg
width={size}
height={size}
class={className}
viewBox="0 0 13 13"
style="fill:currentcolor;"
xmlns="http://www.w3.org/2000/svg"
@@ -183,7 +183,7 @@
<div class="flex flex-col gap-4 mb-8 mt-20">
<div class="flex flex-col gap-1">
<div class="text-primary text-lg font-semibold">Ducklake</div>
<Description link="https://www.windmill.dev/docs/core_concepts/ducklake">
<Description link="https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake">
Windmill has first class support for Ducklake. You can use and explore ducklakes like a normal
SQL database, even though the data is actually stored in parquet files in S3 !
</Description>
+19 -9
View File
@@ -89,33 +89,43 @@ async function initWasmRuby() {
await initRubyParser(wasmUrlRuby)
}
type InferAssetsResult =
| { status: 'ok'; assets: AssetWithAccessType[] }
| { status: 'error'; error: string }
export async function inferAssets(
language: SupportedLanguage | undefined,
code: string
): Promise<AssetWithAccessType[]> {
): Promise<InferAssetsResult> {
function wrap(raw_result: string): InferAssetsResult {
if (raw_result.startsWith('err:')) {
return { status: 'error', error: raw_result.slice(4).trim() }
}
return { status: 'ok', assets: JSON.parse(raw_result) as AssetWithAccessType[] }
}
try {
if (language === 'duckdb') {
await initWasmRegex()
let r = JSON.parse(parse_assets_sql(code))
return r
return wrap(parse_assets_sql(code))
}
if (language === 'deno' || language === 'nativets' || language === 'bun') {
await initWasmTs()
return JSON.parse(parse_assets_ts(code))
return wrap(parse_assets_ts(code))
}
if (language === 'python3') {
await initWasmPython()
return JSON.parse(parse_assets_py(code))
return wrap(parse_assets_py(code))
}
if (language === 'ansible') {
await initWasmYaml()
return JSON.parse(parse_assets_ansible(code))
return wrap(parse_assets_ansible(code))
}
} catch (e) {
console.error('error parsing assets', e)
return []
return { status: 'error', error: (e as Error)?.message || JSON.stringify(e) }
}
return []
return { status: 'ok', assets: [] }
}
export async function inferAnsibleExecutionMode(code: string): Promise<any> {
+1 -1
View File
@@ -323,7 +323,7 @@ const DUCKDB_INIT_CODE = `-- result_collection=last_statement_all_rows
-- SELECT * FROM db.public.friends;
-- Click the +Ducklake button to use a ducklake
-- https://www.windmill.dev/docs/core_concepts/ducklake
-- https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake
--
-- ATTACH 'ducklake' AS dl;
-- USE dl;
@@ -76,12 +76,7 @@
<tr class="h-14">
<Cell first class="w-16">
<Tooltip>
<AssetGenericIcon
assetKind={asset.kind}
size="24px"
fill=""
class="!fill-secondary !stroke-secondary"
/>
<AssetGenericIcon assetKind={asset.kind} size="16px" class="!text-secondary" />
<svelte:fragment slot="text">{formatAssetKind(asset)}</svelte:fragment>
</Tooltip>
</Cell>