diff --git a/backend/.sqlx/query-029b81eb00250eacded407b12bcfbab2b3f35354bdb9ef6e30281a4ff6235060.json b/backend/.sqlx/query-029b81eb00250eacded407b12bcfbab2b3f35354bdb9ef6e30281a4ff6235060.json new file mode 100644 index 0000000000..7174024dfe --- /dev/null +++ b/backend/.sqlx/query-029b81eb00250eacded407b12bcfbab2b3f35354bdb9ef6e30281a4ff6235060.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE asset SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "029b81eb00250eacded407b12bcfbab2b3f35354bdb9ef6e30281a4ff6235060" +} diff --git a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json b/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json new file mode 100644 index 0000000000..de10d89550 --- /dev/null +++ b/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6" +} diff --git a/backend/.sqlx/query-534c7337ed8f1bc0e17843d6caeaf7e36c72efaeb90f94fe3b2e53b953ba2f24.json b/backend/.sqlx/query-534c7337ed8f1bc0e17843d6caeaf7e36c72efaeb90f94fe3b2e53b953ba2f24.json new file mode 100644 index 0000000000..ad5cb84e86 --- /dev/null +++ b/backend/.sqlx/query-534c7337ed8f1bc0e17843d6caeaf7e36c72efaeb90f94fe3b2e53b953ba2f24.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE asset SET usage_path = REGEXP_REPLACE(usage_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE usage_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "534c7337ed8f1bc0e17843d6caeaf7e36c72efaeb90f94fe3b2e53b953ba2f24" +} diff --git a/backend/.sqlx/query-6fdab4c131f3126d5020b780ba45927dc778ac6fefdc2f91982f600b7cb9954f.json b/backend/.sqlx/query-6fdab4c131f3126d5020b780ba45927dc778ac6fefdc2f91982f600b7cb9954f.json new file mode 100644 index 0000000000..b9613d15c0 --- /dev/null +++ b/backend/.sqlx/query-6fdab4c131f3126d5020b780ba45927dc778ac6fefdc2f91982f600b7cb9954f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6fdab4c131f3126d5020b780ba45927dc778ac6fefdc2f91982f600b7cb9954f" +} diff --git a/backend/.sqlx/query-74a2871aba7e35527dcefb2538b8cf41c35618f7c15ea047c5082f8feb7a8464.json b/backend/.sqlx/query-74a2871aba7e35527dcefb2538b8cf41c35618f7c15ea047c5082f8feb7a8464.json new file mode 100644 index 0000000000..6a45170165 --- /dev/null +++ b/backend/.sqlx/query-74a2871aba7e35527dcefb2538b8cf41c35618f7c15ea047c5082f8feb7a8464.json @@ -0,0 +1,51 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)\n VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable" + ] + } + } + }, + { + "Custom": { + "name": "asset_access_type", + "kind": { + "Enum": [ + "r", + "w", + "rw" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "74a2871aba7e35527dcefb2538b8cf41c35618f7c15ea047c5082f8feb7a8464" +} diff --git a/backend/.sqlx/query-76033e76f15cee2aa0394d4ec2ff62130e7e48cb40d3b1534b0d791760b33ec7.json b/backend/.sqlx/query-76033e76f15cee2aa0394d4ec2ff62130e7e48cb40d3b1534b0d791760b33ec7.json new file mode 100644 index 0000000000..dc58473a74 --- /dev/null +++ b/backend/.sqlx/query-76033e76f15cee2aa0394d4ec2ff62130e7e48cb40d3b1534b0d791760b33ec7.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n jsonb_build_object(\n 'path', path,\n 'kind', kind,\n 'access_type', usage_access_type\n ) as \"list!: _\"\n FROM asset\n WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3\n ORDER BY path, kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "list!: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "76033e76f15cee2aa0394d4ec2ff62130e7e48cb40d3b1534b0d791760b33ec7" +} diff --git a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json b/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json new file mode 100644 index 0000000000..f7d3361d99 --- /dev/null +++ b/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523" +} diff --git a/backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json b/backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json new file mode 100644 index 0000000000..035e9665e7 --- /dev/null +++ b/backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', asset.path,\n 'kind', asset.kind,\n 'usages', ARRAY_AGG(jsonb_build_object(\n 'path', asset.usage_path,\n 'kind', asset.usage_kind,\n 'access_type', asset.usage_access_type\n )),\n 'metadata', (CASE\n WHEN asset.kind = 'resource' THEN\n jsonb_build_object('resource_type', resource.resource_type)\n ELSE\n NULL\n END\n )\n )) as \"list!: _\"\n FROM asset\n LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1\n WHERE asset.workspace_id = $1\n AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)\n AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))\n AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1))\n GROUP BY asset.path, asset.kind, resource.resource_type\n ORDER BY asset.path, asset.kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "list!: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6" +} diff --git a/backend/.sqlx/query-ff64f77d46fe8e1a07f26634df040ec5ced23cbb4390cd851c3bd7013dfd5758.json b/backend/.sqlx/query-ff64f77d46fe8e1a07f26634df040ec5ced23cbb4390cd851c3bd7013dfd5758.json new file mode 100644 index 0000000000..eb8654534d --- /dev/null +++ b/backend/.sqlx/query-ff64f77d46fe8e1a07f26634df040ec5ced23cbb4390cd851c3bd7013dfd5758.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'flow' AND usage_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ff64f77d46fe8e1a07f26634df040ec5ced23cbb4390cd851c3bd7013dfd5758" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1ea7859997..a10203e23c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8198,6 +8198,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "notify" version = "6.1.1" @@ -14942,6 +14951,7 @@ dependencies = [ "url", "uuid", "windmill-macros", + "windmill-parser", "windmill-parser-py", "windmill-parser-sql", "windmill-parser-ts", @@ -15093,6 +15103,7 @@ version = "1.504.0" dependencies = [ "anyhow", "itertools 0.14.0", + "rustpython-ast", "rustpython-parser", "serde_json", "windmill-parser", @@ -15144,8 +15155,10 @@ version = "1.504.0" dependencies = [ "anyhow", "lazy_static", + "nom 8.0.0", "regex", "regex-lite", + "serde", "serde_json", "windmill-parser", ] diff --git a/backend/migrations/20250625100238_assets.down.sql b/backend/migrations/20250625100238_assets.down.sql new file mode 100644 index 0000000000..26e3cb4464 --- /dev/null +++ b/backend/migrations/20250625100238_assets.down.sql @@ -0,0 +1,4 @@ +DROP TABLE asset; +DROP TYPE ASSET_USAGE_KIND; +DROP TYPE ASSET_ACCESS_TYPE; +DROP TYPE ASSET_KIND; \ No newline at end of file diff --git a/backend/migrations/20250625100238_assets.up.sql b/backend/migrations/20250625100238_assets.up.sql new file mode 100644 index 0000000000..ec9132bde6 --- /dev/null +++ b/backend/migrations/20250625100238_assets.up.sql @@ -0,0 +1,16 @@ +CREATE TYPE ASSET_USAGE_KIND AS ENUM ('script', 'flow'); +CREATE TYPE ASSET_ACCESS_TYPE AS ENUM ('r', 'w', 'rw'); +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable'); + +CREATE TABLE asset ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + path VARCHAR(255) NOT NULL, + kind ASSET_KIND NOT NULL, + usage_access_type ASSET_ACCESS_TYPE, + usage_path VARCHAR(255) NOT NULL, + usage_kind ASSET_USAGE_KIND NOT NULL, + PRIMARY KEY (workspace_id, path, kind, usage_path, usage_kind) +); + +CREATE INDEX idx_asset_usage ON asset (workspace_id, usage_path, usage_kind); +CREATE INDEX idx_asset_kind_path ON asset (workspace_id, kind, path); \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 9111534406..0c50e0f8d8 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -13,4 +13,5 @@ windmill-parser.workspace = true rustpython-parser.workspace = true itertools.workspace = true serde_json.workspace = true -anyhow.workspace = true \ No newline at end of file +anyhow.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs new file mode 100644 index 0000000000..d6f16824c7 --- /dev/null +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -0,0 +1,98 @@ +use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; +use rustpython_parser::{ast::Suite, Parse}; +use windmill_parser::asset_parser::{ + merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, +}; +use AssetUsageAccessType::*; + +pub fn parse_assets(input: &str) -> anyhow::Result>> { + let ast = Suite::parse(input, "main.py") + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + + let mut assets_finder = AssetsFinder { assets: vec![] }; + ast.into_iter() + .for_each(|stmt| assets_finder.visit_stmt(stmt)); + Ok(merge_assets(assets_finder.assets)) +} + +struct AssetsFinder { + assets: Vec>, +} +impl Visitor for AssetsFinder { + // visit_call_expr will not recurse if it detects an asset, + // so this will only be called when no further context was found + fn visit_expr_constant(&mut self, node: ExprConstant) { + match node.value { + Constant::Str(s) => { + if let Some((kind, path)) = parse_asset_syntax(&s) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + }); + } + } + _ => self.generic_visit_expr_constant(node), + } + } + + fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { + match self.visit_expr_call_inner(&node) { + Ok(_) => {} + Err(_) => self.generic_visit_expr_call(node), + } + } +} + +impl AssetsFinder { + fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + let ident: String = node + .func + .as_name_expr() + .and_then(|o| o.id.parse().ok()) + .or_else(|| { + node.func + .as_attribute_expr() + .and_then(|attr| attr.attr.parse().ok()) + }) + .ok_or(())?; + + let (kind, access_type, arg) = match ident.as_str() { + "load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), + "load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), + "write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)), + "get_resource" => (AssetKind::Resource, None, Arg::Pos(0)), + "set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")), + "get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), + "get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)), + "set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)), + _ => return Err(()), + }; + + let arg_val = match arg { + Arg::Pos(i) => node.args.get(i), + Arg::Named(name) => node + .keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .map(|kw| &kw.value), + }; + + match arg_val { + Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { + let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); + self.assets + .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); + } + _ => return Err(()), + }; + Ok(()) + } +} + +enum Arg { + Pos(usize), + Named(&'static str), +} diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 22d2ea6516..9b5736a18d 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -20,6 +20,9 @@ use rustpython_parser::{ Parse, }; +pub mod asset_parser; +pub use asset_parser::parse_assets; + const FUNCTION_CALL: &str = ""; fn filter_non_main(code: &str, main_name: &str) -> String { diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index 085b709faa..e66216404e 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -19,3 +19,5 @@ windmill-parser.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true +serde.workspace = true +nom = "8.0.0" diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs new file mode 100644 index 0000000000..531efd9c50 --- /dev/null +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -0,0 +1,121 @@ +use windmill_parser::asset_parser::{ + merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, +}; +use AssetUsageAccessType::*; + +use nom::{ + branch::alt, + bytes::complete::{tag, tag_no_case, take_while}, + character::complete::{char, multispace0}, + IResult, Parser, +}; + +pub fn parse_assets<'a>(input: &str) -> anyhow::Result>> { + let mut assets = Vec::new(); + let mut remaining = input; + + while !remaining.trim().is_empty() { + if let Ok((rest, _)) = parse_comment(remaining) { + remaining = rest; // skip comment + } + if let Ok((rest, res)) = parse_asset(remaining) { + assets.push(res); + remaining = rest; + } else { + remaining = &remaining[1..]; // skip 1 char and continue + } + } + + Ok(merge_assets(assets)) +} + +fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> { + alt(( + parse_s3_object_read.map(|path| ParseAssetsResult { + path, + kind: AssetKind::S3Object, + access_type: Some(R), + }), + parse_s3_object_write.map(|path| ParseAssetsResult { + path, + kind: AssetKind::S3Object, + access_type: Some(W), + }), + // Parse ambiguous access_types at the end if we could not find precisely read or copy + parse_s3_object_lit.map(|path| ParseAssetsResult { + path, + kind: AssetKind::S3Object, + access_type: None, + }), + parse_resource_lit.map(|path| ParseAssetsResult { + path, + kind: AssetKind::Resource, + access_type: None, + }), + )) + .parse(input) +} + +/// Any expression that reads an s3 asset +fn parse_s3_object_read(input: &str) -> IResult<&str, &str> { + alt((parse_s3_object_read_fn, parse_s3_object_select_from)).parse(input) +} + +/// Any expression that writes to an s3 asset +fn parse_s3_object_write(input: &str) -> IResult<&str, &str> { + // COPY (...) TO 's3://...' + let (input, _) = (tag_no_case("TO"), multispace0).parse(input)?; + let (input, path) = parse_s3_object_lit(input)?; + Ok((input, path)) +} + +/// read_parquet('s3://...') +fn parse_s3_object_read_fn(input: &str) -> IResult<&str, &str> { + let (input, _) = alt(( + tag_no_case("read_parquet"), + tag_no_case("read_csv"), + tag_no_case("read_json"), + )) + .parse(input)?; + let (input, _) = multispace0(input)?; + let (input, _) = char('(')(input)?; + let (input, _) = multispace0(input)?; + let (input, path) = parse_s3_object_lit(input)?; + let (input, _) = multispace0(input)?; + let (input, _) = char(')')(input)?; + Ok((input, path)) +} + +/// SELECT ... FROM 's3://...' +fn parse_s3_object_select_from(input: &str) -> IResult<&str, &str> { + let (input, _) = tag_no_case("FROM").parse(input)?; + let (input, _) = multispace0(input)?; + let (input, path) = parse_s3_object_lit(input)?; + Ok((input, path)) +} +/// 's3://...' +fn parse_s3_object_lit(input: &str) -> IResult<&str, &str> { + let (input, _) = quote(input)?; + let (input, _) = tag("s3://").parse(input)?; + let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?; + let (input, _) = quote(input)?; + Ok((input, path)) +} + +fn quote(input: &str) -> IResult<&str, char> { + alt((char('\''), char('\"'))).parse(input) +} + +fn parse_resource_lit(input: &str) -> IResult<&str, &str> { + let (input, _) = quote(input)?; + let (input, _) = alt((tag("$res:"), tag("res://"))).parse(input)?; + let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?; + let (input, _) = quote(input)?; + Ok((input, path)) +} + +fn parse_comment(input: &str) -> IResult<&str, &str> { + let (input, _) = tag("--").parse(input)?; + let (input, comment) = take_while(|c| c != '\n')(input)?; + Ok((input, comment)) +} diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 0fe72f0296..ac34a7de38 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -19,6 +19,9 @@ pub use windmill_parser::{Arg, MainArgSignature, Typ}; pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__"; pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__"; +mod asset_parser; +pub use asset_parser::parse_assets; + pub fn parse_mysql_sig(code: &str) -> anyhow::Result { let parsed = parse_mysql_file(&code)?; if let Some(x) = parsed { diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs new file mode 100644 index 0000000000..dcd6681f5f --- /dev/null +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -0,0 +1,104 @@ +use swc_common::{sync::Lrc, FileName, SourceMap}; +use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, Str}; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; +use swc_ecma_visit::{Visit, VisitWith}; +use windmill_parser::asset_parser::{ + merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, +}; +use AssetUsageAccessType::*; + +pub fn parse_assets(code: &str) -> anyhow::Result>> { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into()); + let lexer = Lexer::new( + // We want to parse ecmascript + Syntax::Typescript(TsSyntax::default()), + // EsVersion defaults to es5 + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + + let mut err_s = "".to_string(); + for e in parser.take_errors() { + err_s += &e.into_kind().msg().to_string(); + } + + let ast = parser + .parse_module() + .map_err(|e| { + anyhow::anyhow!("Error while parsing code, it is invalid TypeScript: {err_s}, {e:?}") + })? + .body; + let mut assets_finder = AssetsFinder { assets: vec![] }; + assets_finder.visit_module_items(&ast); + Ok(merge_assets(assets_finder.assets)) +} + +struct AssetsFinder { + assets: Vec>, +} + +impl Visit for AssetsFinder { + // visit_call_expr will not recurse if it detects an asset, + // so this will only be called when no further context was found + fn visit_lit(&mut self, node: &swc_ecma_ast::Lit) { + match node { + swc_ecma_ast::Lit::Str(str) => { + if let Some((kind, path)) = parse_asset_syntax(str.value.as_str()) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + }); + } + } + _ => >::visit_children_with(node, self), + } + } + + fn visit_call_expr(&mut self, node: &swc_ecma_ast::CallExpr) { + match self.visit_call_expr_inner(node) { + Ok(_) => {} + Err(_) => >::visit_children_with(node, self), + } + } +} + +impl AssetsFinder { + fn visit_call_expr_inner(&mut self, node: &swc_ecma_ast::CallExpr) -> Result<(), ()> { + let ident = match node.callee.as_expr().map(AsRef::as_ref) { + Some(Expr::Ident(i)) => i.sym.as_str(), + Some(Expr::Member(MemberExpr { prop: MemberProp::Ident(i), .. })) => i.sym.as_str(), + _ => return Err(()), + }; + let (kind, access_type, arg_pos) = match ident { + "loadS3File" => (AssetKind::S3Object, Some(R), 0), + "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0), + "writeS3File" => (AssetKind::S3Object, Some(W), 0), + "getResource" => (AssetKind::Resource, None, 0), + "setResource" => (AssetKind::Resource, Some(W), 1), + "databaseUrlFromResource" => (AssetKind::Resource, None, 0), + "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), + "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), + "polarsConnectionSettings" => (AssetKind::Resource, None, 0), + "getVariable" => (AssetKind::Variable, Some(R), 0), + "setVariable" => (AssetKind::Variable, Some(W), 0), + _ => return Err(()), + }; + + let arg_value = node.args.get(arg_pos); + + match arg_value.map(|e| e.expr.as_ref()) { + Some(Expr::Lit(Lit::Str(Str { value, .. }))) => { + let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); + self.assets + .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); + } + _ => return Err(()), + } + Ok(()) + } +} diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 8ace23a1d3..d66bc8f0b5 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -159,6 +159,9 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result> { Ok(visitor.idents.into_iter().collect()) } +pub mod asset_parser; +pub use asset_parser::parse_assets; + /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. pub fn parse_deno_signature( diff --git a/backend/parsers/windmill-parser-wasm/package-lock.json b/backend/parsers/windmill-parser-wasm/package-lock.json deleted file mode 100644 index f88d35a75a..0000000000 --- a/backend/parsers/windmill-parser-wasm/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "windmill-parser-wasm", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/backend/parsers/windmill-parser-wasm/package.json b/backend/parsers/windmill-parser-wasm/package.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/backend/parsers/windmill-parser-wasm/package.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2ca730cf72..e02e280abb 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -168,4 +168,34 @@ pub fn parse_java(code: &str) -> String { wrap_sig(windmill_parser_java::parse_java_signature(code)) } +#[cfg(feature = "sql-parser")] +#[wasm_bindgen] +pub fn parse_assets_sql(code: &str) -> String { + if let Ok(r) = windmill_parser_sql::parse_assets(code) { + return serde_json::to_string(&r).unwrap(); + } else { + return "Invalid".to_string(); + } +} + +#[cfg(feature = "ts-parser")] +#[wasm_bindgen] +pub fn parse_assets_ts(code: &str) -> String { + if let Ok(r) = windmill_parser_ts::parse_assets(code) { + return serde_json::to_string(&r).unwrap(); + } else { + return "Invalid".to_string(); + } +} + +#[cfg(feature = "py-parser")] +#[wasm_bindgen] +pub fn parse_assets_py(code: &str) -> String { + if let Ok(r) = windmill_parser_py::parse_assets(code) { + return serde_json::to_string(&r).unwrap(); + } else { + return "Invalid".to_string(); + } +} + // for related places search: ADD_NEW_LANG diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs new file mode 100644 index 0000000000..893843342b --- /dev/null +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -0,0 +1,65 @@ +use serde::Serialize; + +#[derive(Serialize, PartialEq, Clone, Copy)] +#[serde(rename_all(serialize = "lowercase"))] +pub enum AssetUsageAccessType { + R, + W, + RW, +} + +use AssetUsageAccessType::*; + +#[derive(Serialize, PartialEq, Clone, Copy)] +#[serde(rename_all(serialize = "lowercase"))] +pub enum AssetKind { + S3Object, + Resource, + Variable, +} + +#[derive(Serialize)] +pub struct ParseAssetsResult> { + pub kind: AssetKind, + pub path: S, + #[serde(skip_serializing_if = "Option::is_none")] + pub access_type: Option, // None in case of ambiguity +} + +pub fn merge_assets>(assets: Vec>) -> Vec> { + let mut arr: Vec> = vec![]; + for asset in assets { + // Remove duplicates + if let Some(existing) = arr + .iter_mut() + .find(|x| x.path.as_ref() == asset.path.as_ref() && x.kind == asset.kind) + { + // merge access types + existing.access_type = match (asset.access_type, existing.access_type) { + (None, _) | (_, None) => None, + (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), + (Some(RW), _) | (_, Some(RW)) => Some(RW), + (Some(R), Some(R)) => Some(R), + (Some(W), Some(W)) => Some(W), + }; + } else { + arr.push(asset); + } + } + arr.sort_by(|a, b| a.path.as_ref().cmp(b.path.as_ref())); + arr +} + +pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { + if s.starts_with("s3://") { + Some((AssetKind::S3Object, &s[5..])) + } else if s.starts_with("res://") { + Some((AssetKind::Resource, &s[6..])) + } else if s.starts_with("$res:") { + Some((AssetKind::Resource, &s[5..])) + } else if s.starts_with("var://") { + Some((AssetKind::Variable, &s[6..])) + } else { + None + } +} diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 0cb662c91b..8a3ac91555 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -10,6 +10,8 @@ use convert_case::{Boundary, Case, Casing}; use serde::Serialize; use serde_json::Value; +pub mod asset_parser; + #[derive(Serialize, Debug, PartialEq, Default)] pub struct MainArgSignature { pub star_args: bool, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 9403df9f08..4bce2756da 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1147,6 +1147,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, } .into(), stop_after_if: Default::default(), @@ -1189,6 +1190,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, } .into(), stop_after_if: Default::default(), @@ -1316,6 +1318,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, }.into(), stop_after_if: Default::default(), @@ -1369,6 +1372,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), @@ -1407,6 +1411,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, }.into(), stop_after_if: Default::default(), @@ -1470,6 +1475,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d7992e4ace..7379428089 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4971,7 +4971,8 @@ paths: /scripts_u/tokened_raw/{workspace}/{token}/{path}: get: - summary: raw script by path with a token (mostly used by lsp to be used with + summary: + raw script by path with a token (mostly used by lsp to be used with import maps to resolve scripts) operationId: rawScriptByPathTokened tags: @@ -6772,7 +6773,8 @@ paths: schema: type: string - name: branch_or_iteration_n - description: for branchall or loop, the iteration at which the flow should + description: + for branchall or loop, the iteration at which the flow should restart required: true in: path @@ -8490,7 +8492,7 @@ paths: text/plain: schema: type: string - + /w/{workspace}/openapi/download: post: summary: Download the OpenAPI v3.1 spec as a file @@ -9845,7 +9847,6 @@ paths: schema: type: string - /w/{workspace}/gcp_triggers/subscriptions/delete/{path}: delete: summary: delete gcp trigger @@ -9931,7 +9932,7 @@ paths: application/json: schema: type: string - + /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}: get: summary: check if postgres configuration is set to logical @@ -11383,7 +11384,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] responses: "200": @@ -11427,7 +11428,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] requestBody: description: acl to add @@ -11482,7 +11483,7 @@ paths: postgres_trigger, mqtt_trigger, gcp_trigger, - sqs_trigger + sqs_trigger, ] requestBody: description: acl to add @@ -11861,7 +11862,8 @@ paths: /w/{workspace}/job_helpers/duckdb_connection_settings: post: - summary: Converts an S3 resource to the set of instructions necessary to connect + summary: + Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettings tags: @@ -11890,7 +11892,8 @@ paths: type: string /w/{workspace}/job_helpers/v2/duckdb_connection_settings: post: - summary: Converts an S3 resource to the set of instructions necessary to connect + summary: + Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettingsV2 tags: @@ -11926,7 +11929,8 @@ paths: /w/{workspace}/job_helpers/polars_connection_settings: post: - summary: Converts an S3 resource to the set of arguments necessary to connect + summary: + Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettings tags: @@ -11970,7 +11974,8 @@ paths: - client_kwargs /w/{workspace}/job_helpers/v2/polars_connection_settings: post: - summary: Converts an S3 resource to the set of arguments necessary to connect + summary: + Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettingsV2 tags: @@ -12047,7 +12052,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" requestBody: - description: S3 resource path to use. If empty, the S3 resource defined in the + description: + S3 resource path to use. If empty, the S3 resource defined in the workspace settings will be used required: true content: @@ -12960,6 +12966,94 @@ paths: schema: type: string + /w/{workspace}/assets/list: + get: + summary: List all assets in the workspace + operationId: listAssets + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: all assets in the workspace + content: + application/json: + schema: + type: array + items: + type: object + required: [path, kind, usages] + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetKind" + usages: + type: array + items: + type: object + required: [path, kind] + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetUsageKind" + access_type: + $ref: "#/components/schemas/AssetUsageAccessType" + metadata: + type: object + properties: + resource_type: + type: string + + /w/{workspace}/assets/list_by_usages: + post: + summary: List all assets used by given usages paths + operationId: listAssetsByUsage + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: list assets by usages + required: true + content: + application/json: + schema: + type: object + required: [usages] + properties: + usages: + type: array + items: + type: object + required: [path, kind] + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetUsageKind" + responses: + "200": + description: all assets used by the given usage paths, in the same order + content: + application/json: + schema: + type: array + items: + type: array + items: + type: object + required: [path, kind] + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetKind" + access_type: + $ref: "#/components/schemas/AssetUsageAccessType" + components: securitySchemes: bearerAuth: @@ -13106,7 +13200,8 @@ components: type: string ParentJob: name: parent_job - description: The parent job that is at the origin and responsible for the execution + description: + The parent job that is at the origin and responsible for the execution of this script if any in: query schema: @@ -13126,7 +13221,8 @@ components: type: string NewJobId: name: job_id - description: The job id to assign to the created job. if missing, job is chosen + description: + The job id to assign to the created job. if missing, job is chosen randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query @@ -13217,7 +13313,8 @@ components: format: date-time CreatedOrStartedAfter: name: created_or_started_after - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query schema: @@ -13225,7 +13322,8 @@ components: format: date-time CreatedOrStartedAfterCompletedJob: name: created_or_started_after_completed_jobs - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query schema: @@ -13233,7 +13331,8 @@ components: format: date-time CreatedOrStartedBefore: name: created_or_started_before - description: filter on created_at for non non started job and started_at otherwise + description: + filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query schema: @@ -13321,7 +13420,8 @@ components: enum: [Create, Update, Delete, Execute] JobKinds: name: job_kinds - description: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') + description: + filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query schema: @@ -13416,7 +13516,19 @@ components: AIProvider: type: string - enum: [openai, azure_openai, anthropic, mistral, deepseek, googleai, groq, openrouter, togetherai, customai] + enum: + [ + openai, + azure_openai, + anthropic, + mistral, + deepseek, + googleai, + groq, + openrouter, + togetherai, + customai, + ] AIProviderModel: type: object @@ -13483,7 +13595,7 @@ components: alerts: type: array items: - $ref: '#/components/schemas/Alert' + $ref: "#/components/schemas/Alert" Script: type: object @@ -13657,6 +13769,21 @@ components: type: boolean on_behalf_of_email: type: string + fallback_access_types: + type: array + items: + type: object + required: [path, kind, access_type] + properties: + path: + type: string + kind: + type: string + enum: [s3object, resource] + access_type: + type: string + enum: [r, w, rw] + required: - path - summary @@ -13804,7 +13931,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript" + "appscript", ] schedule_path: type: string @@ -13911,7 +14038,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript" + "appscript", ] schedule_path: type: string @@ -14363,7 +14490,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14403,7 +14530,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14429,7 +14556,7 @@ components: "bytes", "dict", "datetime", - "sql" + "sql", ] - type: object properties: @@ -14482,7 +14609,7 @@ components: csharp, nu, java, - duckdb + duckdb, # for related places search: ADD_NEW_LANG ] @@ -14957,7 +15084,7 @@ components: - user_or_folder_regex_value - path - runnable_kind - + OpenapiV3Info: type: object properties: @@ -15537,7 +15664,6 @@ components: - delivery_type - subscription_mode - SubscriptionMode: type: string enum: @@ -15545,7 +15671,6 @@ components: - create_update description: "The mode of subscription. 'existing' means using an existing GCP subscription, while 'create_update' involves creating or updating a new subscription." - GcpTriggerData: type: object properties: @@ -15587,7 +15712,6 @@ components: required: - topic_id - DeleteGcpSubscription: type: object properties: @@ -16583,7 +16707,14 @@ components: properties: type: type: string - enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] + enum: + [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc", + "GoogleCloudStorage", + ] s3_resource_path: type: string azure_blob_resource_path: @@ -16600,7 +16731,13 @@ components: type: type: string enum: - ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] + [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc", + "GoogleCloudStorage", + ] s3_resource_path: type: string azure_blob_resource_path: @@ -16990,7 +17127,8 @@ components: CaptureTriggerKind: type: string - enum: [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp] + enum: + [webhook, http, websocket, kafka, email, nats, postgres, sqs, mqtt, gcp] Capture: type: object @@ -17031,6 +17169,7 @@ components: - runs - schedules - resources + - assets - variables - triggers - audit_logs @@ -17050,6 +17189,9 @@ components: variables: type: boolean description: Whether operators can view variables + assets: + type: boolean + description: Whether operators can view assets audit_logs: type: boolean description: Whether operators can view audit logs @@ -17188,3 +17330,28 @@ components: type: string description: Microsoft Teams channel name minLength: 1 + AssetUsageKind: + type: string + enum: + - script + - flow + AssetUsageAccessType: + type: string + enum: + - r + - w + - rw + AssetKind: + type: string + enum: + - s3object + - resource + - variable + Asset: + type: object + properties: + path: + type: string + kind: + $ref: "#/components/schemas/AssetKind" + required: [path, kind] diff --git a/backend/windmill-api/src/assets.rs b/backend/windmill-api/src/assets.rs new file mode 100644 index 0000000000..196ed0cfee --- /dev/null +++ b/backend/windmill-api/src/assets.rs @@ -0,0 +1,96 @@ +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; +use serde::Deserialize; +use serde_json::Value; +use windmill_common::{assets::AssetUsageKind, db::UserDB, error::JsonResult}; + +use crate::db::ApiAuthed; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_assets)) + .route("/list_by_usages", post(list_assets_by_usages)) +} + +async fn list_assets( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let assets = sqlx::query_scalar!( + r#"SELECT + jsonb_strip_nulls(jsonb_build_object( + 'path', asset.path, + 'kind', asset.kind, + 'usages', ARRAY_AGG(jsonb_build_object( + 'path', asset.usage_path, + 'kind', asset.usage_kind, + 'access_type', asset.usage_access_type + )), + 'metadata', (CASE + WHEN asset.kind = 'resource' THEN + jsonb_build_object('resource_type', resource.resource_type) + ELSE + NULL + END + ) + )) as "list!: _" + FROM asset + LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1 + WHERE asset.workspace_id = $1 + AND (asset.kind <> 'resource' OR resource.path IS NOT NULL) + AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1)) + AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1)) + GROUP BY asset.path, asset.kind, resource.resource_type + ORDER BY asset.path, asset.kind"#, + w_id, + ) + .fetch_all(&mut *user_db.begin(&authed).await?) + .await?; + + Ok(Json(assets)) +} + +#[derive(Deserialize)] +pub struct ListAssetsByUsagesBodyInner { + kind: AssetUsageKind, + path: String, +} + +#[derive(Deserialize)] +struct ListAssetsByUsagesBody { + usages: Vec, +} + +async fn list_assets_by_usages( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Json(body): Json, +) -> JsonResult>> { + let mut tx = user_db.begin(&authed).await?; + let mut assets_vec = vec![]; + for usage in body.usages { + let assets = sqlx::query_scalar!( + r#"SELECT + jsonb_build_object( + 'path', path, + 'kind', kind, + 'access_type', usage_access_type + ) as "list!: _" + FROM asset + WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3 + ORDER BY path, kind"#, + w_id, + usage.path, + usage.kind as AssetUsageKind + ) + .fetch_all(&mut *tx) + .await?; + assets_vec.push(assets); + } + Ok(Json(assets_vec)) +} diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 40f87a1f94..f420f2e074 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1168,6 +1168,14 @@ async fn archive_flow_by_path( .execute(&mut *tx) .await?; + sqlx::query!( + "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'flow' AND usage_path = $2", + &w_id, + path + ) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, @@ -1370,6 +1378,7 @@ mod tests { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, + asset_fallback_access_types: None, }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 7c274af630..38bd67e675 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -72,6 +72,7 @@ mod agent_workers_oss; mod ai; mod apps; pub mod args; +mod assets; mod audit; pub mod auth; mod capture; @@ -564,6 +565,7 @@ pub async fn run_server( // Reordered alphabetically .nest("/acls", granular_acls::workspaced_service()) .nest("/apps", apps::workspaced_service()) + .nest("/assets", assets::workspaced_service()) .nest("/audit", audit::workspaced_service()) .nest("/capture", capture::workspaced_service()) .nest( diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 83773d4087..4887f9e6ac 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -42,7 +42,11 @@ use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; -use windmill_common::{error::to_anyhow, worker::CLOUD_HOSTED}; +use windmill_common::{ + assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind}, + error::to_anyhow, + worker::CLOUD_HOSTED, +}; use windmill_common::{ db::UserDB, @@ -659,6 +663,15 @@ async fn create_script_internal<'c>( ) .execute(&mut *tx) .await?; + + sqlx::query!( + "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + &w_id, + p_hash.0 + ) + .execute(&mut *tx) + .await?; + r } }?; @@ -908,6 +921,19 @@ async fn create_script_internal<'c>( ); } + clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; + for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() { + insert_asset_usage( + &mut *tx, + &w_id, + asset, + ns.fallback_access_types.as_ref().map(Vec::as_slice), + &ns.path, + AssetUsageKind::Script, + ) + .await?; + } + let permissioned_as = username_to_permissioned_as(&authed.username); if needs_lock_gen { let tag = if ns.dedicated_worker.is_some_and(|x| x) { @@ -1552,6 +1578,15 @@ async fn archive_script_by_path( .fetch_one(&db) .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; + + sqlx::query!( + "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = $2", + &w_id, + path + ) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, @@ -1603,6 +1638,14 @@ async fn archive_script_by_hash( .await .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; + sqlx::query!( + "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + &w_id, + &hash.0 + ) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, @@ -1643,6 +1686,14 @@ async fn delete_script_by_hash( .await .map_err(|e| Error::internal_err(format!("deleting script by hash {w_id}: {e:#}")))?; + sqlx::query!( + "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + &w_id, + hash.0 + ) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 99296c6689..770e6c793a 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -1280,10 +1280,7 @@ async fn join_workspace<'c>( Ok((tx, username)) } -async fn leave_instance( - Extension(db): Extension, - authed: ApiAuthed, -) -> Result { +async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Result { let mut tx = db.begin().await?; sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) .execute(&mut *tx) @@ -2643,6 +2640,15 @@ async fn update_username_in_workpsace<'c>( ).execute(&mut **tx) .await?; + sqlx::query!( + r#"UPDATE asset SET usage_path = REGEXP_REPLACE(usage_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE usage_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( r#"UPDATE flow_node SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, new_username, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index d75da1fb40..568f016479 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2195,6 +2195,7 @@ struct ChangeOperatorSettings { schedules: bool, resources: bool, variables: bool, + assets: bool, triggers: bool, audit_logs: bool, groups: bool, diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 6238ead74f..8a92a3a05b 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -198,6 +198,14 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE asset SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( "UPDATE flow_node SET workspace_id = $1 WHERE workspace_id = $2", &rw.new_id, diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index d98897d8cb..980ec87d64 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -75,6 +75,7 @@ windmill-macros.workspace = true windmill-parser-sql.workspace = true windmill-parser-ts.workspace = true windmill-parser-py.workspace = true +windmill-parser.workspace = true jsonwebtoken.workspace = true backon.workspace = true openidconnect = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs new file mode 100644 index 0000000000..6b720b368d --- /dev/null +++ b/backend/windmill-common/src/assets.rs @@ -0,0 +1,148 @@ +use serde::{Deserialize, Serialize}; +use sqlx::PgExecutor; +use windmill_parser::asset_parser::ParseAssetsResult; + +use crate::{error, scripts::ScriptLang}; + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetKind { + S3Object, + Resource, + Variable, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageKind { + Script, + Flow, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_ACCESS_TYPE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageAccessType { + R, + W, + RW, +} + +pub struct Asset { + pub path: String, + pub kind: AssetKind, +} + +pub struct AssetUsage { + pub path: String, + pub kind: AssetUsageKind, + pub access_type: AssetUsageAccessType, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Hash)] +pub struct AssetWithAccessType { + pub path: String, + pub kind: AssetKind, + pub access_type: AssetUsageAccessType, +} + +pub fn parse_assets( + input: &str, + lang: ScriptLang, +) -> anyhow::Result>>> { + let r = match lang { + ScriptLang::Python3 => windmill_parser_py::parse_assets(input), + ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| { + a.iter() + .map(|a| ParseAssetsResult { + path: a.path.to_string(), + access_type: a.access_type, + kind: a.kind, + }) + .collect() + }), + ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => { + windmill_parser_ts::parse_assets(input) + } + _ => return Ok(None), + }; + return r.map(Some); +} + +impl From for AssetKind { + fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self { + match kind { + windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, + windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, + windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable, + } + } +} +impl From for AssetUsageAccessType { + fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { + match access_type { + windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, + windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, + windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, + } + } +} + +pub async fn insert_asset_usage<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + parsed_asset: &ParseAssetsResult, + fallback_access_types: Option<&[AssetWithAccessType]>, + usage_path: &str, + usage_kind: AssetUsageKind, +) -> error::Result<()> { + let kind: AssetKind = parsed_asset.kind.into(); + let asset_alternative_access_type = || { + fallback_access_types + .as_ref() + .and_then(|v| { + v.iter() + .find(|a| a.kind == kind && a.path == parsed_asset.path) + }) + .map(|a| a.access_type) + }; + let access_type: Option = parsed_asset + .access_type + .map(Into::into) + .or_else(asset_alternative_access_type); + + sqlx::query!( + r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#, + workspace_id, + parsed_asset.path, + kind as AssetKind, + access_type as Option, + usage_path, + usage_kind as AssetUsageKind + ) + .execute(executor) + .await?; + + Ok(()) +} + +pub async fn clear_asset_usage<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + usage_path: &str, + usage_kind: AssetUsageKind, +) -> error::Result<()> { + sqlx::query!( + r#"DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3"#, + workspace_id, + usage_path, + usage_kind as AssetUsageKind + ) + .execute(executor) + .await?; + + Ok(()) +} diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 443461af64..377653ff36 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -18,6 +18,7 @@ use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ + assets::AssetWithAccessType, cache, error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, @@ -502,6 +503,8 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + asset_fallback_access_types: Option>, }, Identity, // Internal only, never exposed to the frontend. @@ -555,6 +558,7 @@ struct UntaggedFlowModuleValue { id: Option, default_node: Option, modules_node: Option, + asset_fallback_access_types: Option>, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -629,6 +633,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, + asset_fallback_access_types: untagged.asset_fallback_access_types, }), "flowscript" => Ok(FlowModuleValue::FlowScript { input_transforms: untagged.input_transforms.unwrap_or_default(), @@ -800,6 +805,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, + asset_fallback_access_types: None, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 498beecd6c..b619891037 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -26,6 +26,7 @@ use sqlx::{Pool, Postgres}; pub mod agent_workers; pub mod apps; +pub mod assets; pub mod auth; #[cfg(feature = "benchmark")] pub mod bench; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index ffa7bd486a..57dd7d2498 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -13,6 +13,7 @@ use std::{ }; use crate::{ + assets::AssetWithAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, @@ -348,6 +349,7 @@ pub struct NewScript { pub codebase: Option, pub has_preprocessor: Option, pub on_behalf_of_email: Option, + pub fallback_access_types: Option>, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 8ee78292d3..f2af992116 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -431,14 +431,14 @@ struct ParsedAttachDbResource<'a> { } fn parse_attach_db_resource<'a>(query: &'a str) -> Option> { lazy_static::lazy_static! { - static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '\$res:([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap(); + static ref RE: regex::Regex = regex::Regex::new(r"ATTACH '(\$res:|res://)([^']+)' AS (\S+) \(TYPE (\w+)(.*)\)").unwrap(); } for cap in RE.captures_iter(query) { if let (Some(resource_path), Some(name), Some(db_type)) = - (cap.get(1), cap.get(2), cap.get(3)) + (cap.get(2), cap.get(3), cap.get(4)) { - let extra_args = cap.get(4).map(|m| query[m.start()..m.end()].trim()); + let extra_args = cap.get(5).map(|m| query[m.start()..m.end()].trim()); return Some(ParsedAttachDbResource { resource_path: query[resource_path.start()..resource_path.end()].trim(), name: query[name.start()..name.end()].trim(), diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index e24ca1972a..ebdb155465 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -11,6 +11,9 @@ use serde_json::{json, Value}; use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; +use windmill_common::assets::{ + clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind, +}; use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; @@ -718,6 +721,8 @@ pub async fn handle_flow_dependency_job( .execute(&mut *tx) .await?; } + clear_asset_usage(&mut *tx, &job.workspace_id, &job_path, AssetUsageKind::Flow).await?; + let modified_ids; let errors; (flow.modules, tx, modified_ids, errors) = lock_modules( @@ -929,6 +934,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + asset_fallback_access_types, } = e.get_value()? else { match e.get_value()? { @@ -1116,6 +1122,18 @@ async fn lock_modules<'c>( continue; }; + for asset in parse_assets(&content, language)?.iter().flatten() { + insert_asset_usage( + &mut *tx, + &job.workspace_id, + asset, + asset_fallback_access_types.as_ref().map(Vec::as_slice), + job_path, + AssetUsageKind::Flow, + ) + .await?; + } + if let Some(locks_to_reload) = locks_to_reload { if !locks_to_reload.contains(&e.id) { new_flow_modules.push(e); @@ -1211,6 +1229,7 @@ async fn lock_modules<'c>( None } }; + e.value = windmill_common::worker::to_raw_value(&FlowModuleValue::RawScript { lock, path, @@ -1222,8 +1241,10 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + asset_fallback_access_types, }); new_flow_modules.push(e); + continue; } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e25277d665..fa7ca9ee4a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -78,10 +78,10 @@ "windmill-parser-wasm-java": "^1.478.1", "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.499.0", - "windmill-parser-wasm-regex": "^1.492.1", + "windmill-parser-wasm-py": "^1.504.0", + "windmill-parser-wasm-regex": "^1.504.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.486.1", + "windmill-parser-wasm-ts": "^1.504.0", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "xterm": "^5.3.0", @@ -12963,14 +12963,14 @@ "integrity": "sha512-SGJAtNpfdRZftkGboxWsm/yQDnJBJodwPQUbX2cWk/aoNook6ULesZwsYtBC9WN1VH6TIskLiVPohMmu6jtXmw==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.499.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.499.0.tgz", - "integrity": "sha512-ur5SU+YTuYm1Hgo+SWohYRsmkVxIZPME7GzzRivPhkK5XBQcc7PsxlMjYkIvFJwufz+jX2t6p0aYNIdDXpMNbA==" + "version": "1.504.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.504.0.tgz", + "integrity": "sha512-LHMqrR6zQXFCHez6XExztoMe00v8x4HCDws05m9RN8+XZdAqqzGNWu0pmNG+c/JDOAPtXwGYADrU/DxECwTf6Q==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.492.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.492.1.tgz", - "integrity": "sha512-CBdjz3x00z2xA4BRZv8/hsuHcpZgL28UKTjYVSj5O9XMYlY9g0LNsEQ5TTl/oVyFBs62/CPyvWwGvW4oSAObzg==" + "version": "1.504.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.504.0.tgz", + "integrity": "sha512-WY9XR8GBSeR3xPm9Dw1F0Kk2wvebZdUeS9obo4roEjsrltguZh10xDC7b5hrvksl9YFMvoWXmgFt9PX4FV/tYg==" }, "node_modules/windmill-parser-wasm-rust": { "version": "1.429.0", @@ -12978,9 +12978,9 @@ "integrity": "sha512-c8mjpiw8RxoaBDtecb+sKeWM/IOjNr4Y06nHudGu8sMM48MNO1LhgcISLv8wl6Z9zWd7OzQrECJ6RLorpii5Uw==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.486.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.486.1.tgz", - "integrity": "sha512-nv7nPpZA5O0Zsve6W2Sm3mxCSxMqg2rfyYHNqozIpBaGhbe8SpgnbyrSaXKsGKSc/bpwpANHE7nVCp23pQUK6Q==" + "version": "1.504.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.504.0.tgz", + "integrity": "sha512-JsNw5I/Kxk4fCh8ScaDQDOqAFaH7r40vuo6rHrD4mqN46fozsUcWMEgecz4JxAtrYlYtogTj+1sP43guY/bf8A==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.429.0", diff --git a/frontend/package.json b/frontend/package.json index eebe149162..4acf2df744 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -145,10 +145,10 @@ "windmill-parser-wasm-java": "^1.478.1", "windmill-parser-wasm-nu": "^1.474.1", "windmill-parser-wasm-php": "^1.429.0", - "windmill-parser-wasm-py": "^1.499.0", - "windmill-parser-wasm-regex": "^1.492.1", + "windmill-parser-wasm-py": "^1.504.0", + "windmill-parser-wasm-regex": "^1.504.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.486.1", + "windmill-parser-wasm-ts": "^1.504.0", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "xterm": "^5.3.0", @@ -541,4 +541,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} \ No newline at end of file +} diff --git a/frontend/src/lib/components/DBManagerDrawerButton.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte similarity index 79% rename from frontend/src/lib/components/DBManagerDrawerButton.svelte rename to frontend/src/lib/components/DBManagerDrawer.svelte index 583d5a71ac..b0da47c4ac 100644 --- a/frontend/src/lib/components/DBManagerDrawerButton.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -4,7 +4,7 @@ import Drawer from './common/drawer/Drawer.svelte' import DrawerContent from './common/drawer/DrawerContent.svelte' import { sendUserToast, sortArray } from '$lib/utils' - import { ArrowLeft, Database, Expand, Loader2, Minimize, RefreshCcw } from 'lucide-svelte' + import { ArrowLeft, Expand, Loader2, Minimize, RefreshCcw } from 'lucide-svelte' import { dbSupportsSchemas, getDbSchemas, @@ -15,7 +15,6 @@ type TableMetadata } from './apps/components/display/dbtable/utils' import DbManager from './DBManager.svelte' - import { Alert } from './common' import { dbDeleteTableActionWithPreviewScript, dbTableOpsWithPreviewScripts } from './dbOps' import { makeCreateTableQuery } from './apps/components/display/dbtable/queries/createTable' import { runScriptAndPollResult } from './jobs/utils' @@ -24,22 +23,24 @@ import SimpleAgTable from './SimpleAgTable.svelte' import { untrack } from 'svelte' - type Props = { - resourceType: DbType - resourcePath: string - class?: string + let resourceType: DbType | undefined = $state(undefined) + let resourcePath: string | undefined = $state(undefined) + let open = $derived(resourcePath && resourceType) + + export function openDrawer(_resourceType: DbType, _resourcePath: string) { + resourceType = _resourceType + resourcePath = _resourcePath + getSchema() + } + export function closeDrawer() { + resourceType = undefined + resourcePath = undefined + refreshCount = 0 + refreshing = false } - let { resourceType, resourcePath }: Props = $props() - let dbSchema: DBSchema | undefined = $derived( - resourcePath in $dbSchemas ? $dbSchemas[resourcePath] : undefined - ) - - let isDrawerOpen: boolean = $state(false) - - let shouldDisplayError = $derived( - resourcePath && resourcePath in $dbSchemas && !$dbSchemas[resourcePath] + resourcePath && resourcePath in $dbSchemas ? $dbSchemas[resourcePath] : undefined ) // `refreshCount` is a derived state. `refreshing` is the source of truth @@ -56,11 +57,11 @@ let expand = $state(false) $effect(() => { - if (!isDrawerOpen) expand = false + if (!open) expand = false }) async function getSchema() { - if ($dbSchemas[resourcePath] && !refreshing) return + if (!resourcePath || !resourceType || ($dbSchemas[resourcePath] && !refreshing)) return try { const oldDbSchema = $dbSchemas[resourcePath] await getDbSchemas( @@ -69,9 +70,7 @@ $workspaceStore, $dbSchemas, (message: string) => { - if (isDrawerOpen) { - sendUserToast(message, true) - } + if (open) sendUserToast(message, true) } ) // avoid infinite loop on error due to the way getDbSchemas is implemented @@ -95,6 +94,8 @@ let cachedLastRefreshCount = 0 async function getColDefs(tableKey: string) { + if (!resourcePath || !resourceType) return [] + if (cachedLastRefreshCount !== refreshCount) cachedColDefs = {} cachedLastRefreshCount = refreshCount if (cachedColDefs[tableKey]) { @@ -130,37 +131,26 @@ }} /> -{#if shouldDisplayError} - - Schema could not be loaded. Please check the permissions of the resource. - -{:else} - - + + {#key [resourceType, resourcePath, dbSchema]} { if (replResultData) { replResultData = undefined } else { - isDrawerOpen = false + closeDrawer() } }} CloseIcon={replResultData ? ArrowLeft : undefined} noPadding > - {#if dbSchema && $workspaceStore} + {#if dbSchema && $workspaceStore && resourceType && resourcePath} @@ -191,8 +181,8 @@ dbTableOpsWithPreviewScripts({ colDefs, tableKey, - resourcePath, - resourceType, + resourcePath: resourcePath!, + resourceType: resourceType!, workspace: $workspaceStore })} dbTableActionsFactory={[ @@ -204,16 +194,16 @@ ]} {refresh} dbTableEditorPropsFactory={({ selectedSchemaKey }) => ({ - resourceType, + resourceType: resourceType!, previewSql: (values) => - makeCreateTableQuery(values, resourceType, selectedSchemaKey), + makeCreateTableQuery(values, resourceType!, selectedSchemaKey), async onConfirm(values) { await runScriptAndPollResult({ workspace: $workspaceStore, requestBody: { args: { database: '$res:' + resourcePath }, - content: makeCreateTableQuery(values, resourceType, selectedSchemaKey), - language: getLanguageByResourceType(resourceType) + content: makeCreateTableQuery(values, resourceType!, selectedSchemaKey), + language: getLanguageByResourceType(resourceType!) } }) refresh() @@ -268,5 +258,5 @@ /> {/snippet} - -{/if} + {/key} + diff --git a/frontend/src/lib/components/DarkModeObserver.svelte b/frontend/src/lib/components/DarkModeObserver.svelte index 606a9891d6..30d0d78091 100644 --- a/frontend/src/lib/components/DarkModeObserver.svelte +++ b/frontend/src/lib/components/DarkModeObserver.svelte @@ -1,30 +1,49 @@ + + diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 60e5349919..41dc59c0f2 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -29,6 +29,7 @@ import { DiffIcon, DollarSign, + File, History, Library, Link, @@ -38,7 +39,7 @@ Save, Users } from 'lucide-svelte' - import { capitalize, toCamel } from '$lib/utils' + import { capitalize, formatS3Object, toCamel } from '$lib/utils' import type { Schema, SchemaProperty, SupportedLanguage } from '$lib/common' import ScriptVersionHistory from './ScriptVersionHistory.svelte' import ScriptGen from './copilot/ScriptGen.svelte' @@ -48,6 +49,7 @@ import ResourceEditorDrawer from './ResourceEditorDrawer.svelte' import type { EditorBarUi } from './custom_ui' import EditorSettings from './EditorSettings.svelte' + import S3FilePicker from './S3FilePicker.svelte' interface Props { lang: SupportedLanguage | 'bunnative' | undefined @@ -109,13 +111,10 @@ let resourceTypePicker: ItemPicker | undefined = $state() let variableEditor: VariableEditor | undefined = $state() let resourceEditor: ResourceEditorDrawer | undefined = $state() - let showContextVarPicker = $state(false) - let showVarPicker = $state(false) - let showResourcePicker = $state(false) - let showResourceTypePicker = $state(false) + let s3FilePicker: S3FilePicker | undefined = $state() - run(() => { - showContextVarPicker = [ + let showContextVarPicker = $derived( + [ 'python3', 'bash', 'powershell', @@ -131,9 +130,10 @@ 'java' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') - }) - run(() => { - showVarPicker = [ + ) + + let showVarPicker = $derived( + [ 'python3', 'bash', 'powershell', @@ -149,9 +149,10 @@ 'java' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') - }) - run(() => { - showResourcePicker = [ + ) + + let showResourcePicker = $derived( + [ 'python3', 'bash', 'powershell', @@ -164,16 +165,22 @@ 'rust', 'csharp', 'nu', - 'java' + 'java', + 'duckdb' // for related places search: ADD_NEW_LANG ].includes(lang ?? '') - }) - run(() => { - showResourceTypePicker = - ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) || + ) + + let showS3Picker = $derived( + ['duckdb', 'python3'].includes(lang ?? '') || + ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) + ) + + let showResourceTypePicker = $derived( + ['typescript', 'javascript'].includes(scriptLangToEditorLang(lang)) || lang === 'python3' || lang === 'php' - }) + ) let codeViewer: Drawer | undefined = $state() let codeObj: { language: SupportedLanguage; content: string } | undefined = $state(undefined) @@ -522,7 +529,7 @@ string ${windmillPathToCamelCaseName(path)} = await client.GetStringAsync(uri); { + pickCallback={(path, _, resType) => { if (!editor) return if (lang == 'deno') { if (!editor.getCode().includes('import * as wmill from')) { @@ -582,6 +589,15 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS } else if (lang == 'java') { editor.insertAtCursor(`(Wmill.getResource("${path}"))`) // for related places search: ADD_NEW_LANG + } else if (lang == 'duckdb') { + let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] + if (!t) { + sendUserToast(`Resource type ${resType} is not supported in DuckDB`, true) + editor.insertAtCursor(`'$res:${path}'`) + return + } else { + editor.insertAtCursor(`ATTACH '$res:${path}' AS db (TYPE ${t});`) + } } sendUserToast(`${path} inserted at cursor`) @@ -628,6 +644,30 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS + { + let s = `'${formatS3Object(s3obj.detail)}'` + if (lang === 'duckdb') { + if (s3obj.detail?.s3.endsWith('.json')) s = `read_json(${s})` + if (s3obj.detail?.s3.endsWith('.csv')) s = `read_csv(${s})` + if (s3obj.detail?.s3.endsWith('.parquet')) s = `read_parquet(${s})` + editor?.insertAtCursor(s) + } else if (lang === 'python3') { + if (!editor?.getCode().includes('import wmill')) { + editor?.insertAtBeginning('import wmill\n') + } + editor?.insertAtCursor(`wmill.load_s3_file(${s})`) + } else if (['javascript', 'typescript'].includes(scriptLangToEditorLang(lang))) { + if (!editor?.getCode().includes('import * as wmill from')) { + editor?.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`) + } + editor?.insertAtCursor(`wmill.loadS3File(${s})`) + } + }} +/> +
{/if} + {#if showS3Picker && customUi?.s3object != false} + + {/if} + {#if showResourcePicker && customUi?.resource != false}
interface Props { - pickCallback: (path: string, f: string) => void + pickCallback: (path: string, extraField: string, extraField2: string) => void loadItems: () => Promise extraField?: string extraField2?: string | undefined @@ -132,7 +132,7 @@ if (closeOnClick) { drawer?.closeDrawer() } - pickCallback(obj['path'], obj[extraField]) + pickCallback(obj['path'], obj[extraField], extraField2 ? obj[extraField2] : '') }} > {#if `app` in obj} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index b07e1f73bf..a20b4b8d3a 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -6,11 +6,13 @@ import ResourceEditorDrawer from './ResourceEditorDrawer.svelte' import { Button } from './common' - import DBManagerDrawerButton from './DBManagerDrawerButton.svelte' import { Pen, Plus, RotateCw } from 'lucide-svelte' import { sendUserToast } from '$lib/toast' - import { isDbType } from './apps/components/display/dbtable/utils' import Select from './select/Select.svelte' + import DbManagerDrawer from './DBManagerDrawer.svelte' + import ExploreAssetButton, { + assetCanBeExplored + } from '../../routes/(root)/(logged)/assets/ExploreAssetButton.svelte' interface Props { initialValue?: string | undefined @@ -142,6 +144,7 @@ let appConnect: AppConnect | undefined = $state() let resourceEditor: ResourceEditorDrawer | undefined = $state() + let dbManagerDrawer: DbManagerDrawer | undefined = $state()
- {#if showSchemaExplorer && isDbType(resourceType) && value} - + {#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'resource', path: value }, { resource_type: resourceType })} + {/if}
+ + diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 5ee8fa8385..59ddac2bef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -16,7 +16,14 @@ import { workspaceStore } from '$lib/stores' import { HelpersService, SettingService } from '$lib/gen' import { base } from '$lib/base' - import { displayDate, displaySize, emptyString, sendUserToast } from '$lib/utils' + import { + displayDate, + displaySize, + emptyString, + parseS3Object, + sendUserToast, + type S3Object + } from '$lib/utils' import { Alert, Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' import Section from './Section.svelte' @@ -66,6 +73,7 @@ let dispatch = createEventDispatcher<{ close: { s3: string; storage: string | undefined } | undefined + selectAndClose: { s3: string; storage: string | undefined } }>() let drawer: Drawer | undefined = $state() @@ -119,9 +127,13 @@ let timeout: NodeJS.Timeout | undefined = undefined let firstLoad = true - let secondaryStorageNames = usePromise(() => - SettingService.getSecondaryStorageNames({ workspace: $workspaceStore! }) + let secondaryStorageNames = usePromise( + () => SettingService.getSecondaryStorageNames({ workspace: $workspaceStore! }), + { loadInit: false } ) + $effect(() => { + $workspaceStore && untrack(() => secondaryStorageNames.refresh()) + }) function onFilterChange() { if (!firstLoad) { @@ -382,9 +394,8 @@ } let storage: string | undefined = $state(undefined) - export async function open( - preSelectedFileKey: { s3: string; storage?: string } | undefined = undefined - ) { + export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { + const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage if (preSelectedFileKey !== undefined) { initialFileKey = { ...preSelectedFileKey } @@ -422,6 +433,9 @@ } async function selectAndClose() { + if (selectedFileKey?.s3) { + dispatch('selectAndClose', { s3: selectedFileKey.s3, storage }) + } drawer?.closeDrawer?.() } diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 31ece0aab1..01a349e994 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -524,7 +524,8 @@ no_main_func: script.no_main_func, has_preprocessor: script.has_preprocessor, deployment_message: deploymentMsg || undefined, - on_behalf_of_email: script.on_behalf_of_email + on_behalf_of_email: script.on_behalf_of_email, + fallback_access_types: script.fallback_access_types } }) @@ -1775,6 +1776,7 @@ kind={script.kind} {template} tag={script.tag} + bind:fallbackAccessTypes={script.fallback_access_types} lastSavedCode={savedScript?.draft?.content} lastDeployedCode={savedScript?.draft_only ? undefined : savedScript?.content} bind:args diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fac6951f91..45277b31ff 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -2,11 +2,18 @@ import { BROWSER } from 'esm-env' import type { Schema, SupportedLanguage } from '$lib/common' - import { type CompletedJob, type Job, JobService, type Preview, type ScriptLang } from '$lib/gen' + import { + AssetService, + type CompletedJob, + type Job, + JobService, + type Preview, + type ScriptLang + } from '$lib/gen' import { copilotInfo, enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils' import Editor from './Editor.svelte' - import { inferArgs } from '$lib/infer' + import { inferArgs, inferAssets } from '$lib/infer' import { Pane, Splitpanes } from 'svelte-splitpanes' import SchemaForm from './SchemaForm.svelte' import LogPanel from './scriptEditor/LogPanel.svelte' @@ -46,6 +53,9 @@ import type { ScriptOptions } from './copilot/chat/ContextManager.svelte' import { aiChatManager, AIMode } from './copilot/chat/AIChatManager.svelte' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' + import AssetsDropdownButton from './assets/AssetsDropdownButton.svelte' + import { usePromise } from '$lib/svelte5Utils.svelte' + import { assetEq, type AssetWithAccessType } from './assets/lib' interface Props { // Exported @@ -77,6 +87,7 @@ lastDeployedCode?: string | undefined disableAi?: boolean editor_bar_right?: import('svelte').Snippet + fallbackAccessTypes?: AssetWithAccessType[] } let { @@ -106,7 +117,8 @@ lastSavedCode = undefined, lastDeployedCode = undefined, disableAi = false, - editor_bar_right + editor_bar_right, + fallbackAccessTypes = $bindable() }: Props = $props() $effect.pre(() => { @@ -135,6 +147,29 @@ dispatch('change', { code, schema }) }) + let parsedAssets = usePromise(() => inferAssets(lang, code), { clearValueOnRefresh: false }) + $effect(() => { + untrack(() => parsedAssets.refresh()), [lang, code] + }) + + // Load initial fallbackAccessTypes + if (edit && path) { + AssetService.listAssetsByUsage({ + workspace: $workspaceStore!, + requestBody: { usages: [{ path, kind: 'script' }] } + }).then((arr) => { + const v = arr[0] + setTimeout(() => { + for (const a of parsedAssets.value ?? []) { + const fallback = v.find((a2) => assetEq(a2, a))?.access_type + if (!a.access_type && fallback) { + fallbackAccessTypes = [...(fallbackAccessTypes ?? []), { ...a, access_type: fallback }] + } + } + }, 200) + }) + } + let width = $state(1200) let testJobLoader: TestJobLoader | undefined = $state(undefined) @@ -506,6 +541,9 @@
+ {#if parsedAssets.value?.length} + + {/if} {#if testPanelSize === 0}