feat: assets as a primary concept (#6125)

* assets migration * parse assets (duckdb) * iterate on assets * S3 object Preview * remove pagination * filterText * better occurence list * tweak * assets in JobPreview * clone impl * AssetsDetectedBadge * improve DbManagerButton + asset dropdown button * edit resource btn * warning when incorrect resource * +Resource in DuckDB * +S3 Object editor bar * nit fix rename * flow asset badge * More Generic OnChange * Highlight assets used in modules * Show occurence count in flow * Better UX, avoid moving parts * nit * Asset nodes * move to dedicated Asset ctx * fix layoutNodes not handling first assetsMap * explore asset btn in flow asset node * correct offset * single computeAssetNodes function * Fix y positioning of nodes with assets * resource editor * write mode node (ui) * accessType in ctx + fix insert button positioning * right positioning when mixing read and write nodes * right positioning when mixing R and W assets * Better layout fix algorithm * listAssetsByUsage and asset nodes on transitive usages * refactor + remove linkAssets * Refactor to allow for custom R/W modes * AssetsDropdownButton in flow script editor * R/W/RW selection and changes node pos in flow * layoutNodes doesnt need recompute now * fix wrong assumption that nodes recompute when assets change * r/w/rw multi toggle * MultiToggle cool animation + clearable * rename + 1px nit * remove mini toggle button group, use ToggleButtonGroup * Combinator parser that detects R / W asset context * nit fix missing flex-1 * missing order by * better ui indication for access type * special x offset case when only one asset node for clarity * parse getResource in TS with swc ecma parser * support load and write s3 detection in TS * Python asset parser * support wmill api calls without special $res: or s3:// syntax * detect out of context asset uris python * do not use access type override when not ambiguous in flow graph * parse_assets match case in rust * AsRef<str> refactor * From impl * Save flow assets * Save script asset usages + fixes + save fallback access types * asset sub icon * max total asset node width to avoid overlap * small refactor * don't parse comments in duckdb assets * fix assets clearing on parse error * fix script asset save in wrong place * load initial asset fallback access types * support variables * ui fixes * Support S3Object as URI in TS client * support new syntax in python client * Support +S3Object in EditorBar for TS and python * Reduce resource requests in assets page * import windmill client when necessary * update s3Types.d.ts * nit fix * Show input resources and s3 objects as assets * improve asset icons * DarkModeObserver refactor * asset page tabs * Moved resource variables and s3object pages to assets tabs * fetch resource usages * Get variables usages * move assets usage dropdown to component * Revert "move assets usage dropdown to component" This reverts commit 622ea4ab12. * Revert "Get variables usages" This reverts commit b11ced4e29. * Revert "fetch resource usages" This reverts commit aa5187ad4b. * Revert "Moved resource variables and s3object pages to assets tabs" This reverts commit 4430487be4. * Revert "asset page tabs" This reverts commit dacc2f0da5. * move assets usage dropdown to component * asset icon in asset pages * tooltip * details * Storage selector in S3 File Picker * make edge less opaque * Refactor computeAssetNodes to separate in and out nodes * AssetsOverflowedNode * nits * fix assets not being parsed in flows sometimes * show asset kind and resource_type * ui nits * support res:// in duckdb * add banner for old deployments * Fix permissionning * fix broken disable /enable all * assets page view permission for operators * Disable ExploreAssetButton for operators * asset kind as subtitle * do not spam getResource in assets page. prob. revert fail * update assets page on workspace change * reload storage names on ws change * delete assets on archive / deletion * sqlx prepare * missing update when updating user * add indexes on asset * better message * missing loadInit: false * dead code * use transaction * typo * update package.json * update package.json --------- Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2026-10-29 16:15:51 +00:00 · 2025-07-11 09:56:53 +00:00
parent bdd3ca1fbe
commit 843ee8d4cb
88 changed files with 3058 additions and 263 deletions
@@ -0,0 +1,121 @@
+use windmill_parser::asset_parser::{
+    merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult,
+};
+use AssetUsageAccessType::*;
+
+use nom::{
+    branch::alt,
+    bytes::complete::{tag, tag_no_case, take_while},
+    character::complete::{char, multispace0},
+    IResult, Parser,
+};
+
+pub fn parse_assets<'a>(input: &str) -> anyhow::Result<Vec<ParseAssetsResult<&str>>> {
+    let mut assets = Vec::new();
+    let mut remaining = input;
+
+    while !remaining.trim().is_empty() {
+        if let Ok((rest, _)) = parse_comment(remaining) {
+            remaining = rest; // skip comment
+        }
+        if let Ok((rest, res)) = parse_asset(remaining) {
+            assets.push(res);
+            remaining = rest;
+        } else {
+            remaining = &remaining[1..]; // skip 1 char and continue
+        }
+    }
+
+    Ok(merge_assets(assets))
+}
+
+fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> {
+    alt((
+        parse_s3_object_read.map(|path| ParseAssetsResult {
+            path,
+            kind: AssetKind::S3Object,
+            access_type: Some(R),
+        }),
+        parse_s3_object_write.map(|path| ParseAssetsResult {
+            path,
+            kind: AssetKind::S3Object,
+            access_type: Some(W),
+        }),
+        // Parse ambiguous access_types at the end if we could not find precisely read or copy
+        parse_s3_object_lit.map(|path| ParseAssetsResult {
+            path,
+            kind: AssetKind::S3Object,
+            access_type: None,
+        }),
+        parse_resource_lit.map(|path| ParseAssetsResult {
+            path,
+            kind: AssetKind::Resource,
+            access_type: None,
+        }),
+    ))
+    .parse(input)
+}
+
+/// Any expression that reads an s3 asset
+fn parse_s3_object_read(input: &str) -> IResult<&str, &str> {
+    alt((parse_s3_object_read_fn, parse_s3_object_select_from)).parse(input)
+}
+
+/// Any expression that writes to an s3 asset
+fn parse_s3_object_write(input: &str) -> IResult<&str, &str> {
+    // COPY (...) TO 's3://...'
+    let (input, _) = (tag_no_case("TO"), multispace0).parse(input)?;
+    let (input, path) = parse_s3_object_lit(input)?;
+    Ok((input, path))
+}
+
+/// read_parquet('s3://...')
+fn parse_s3_object_read_fn(input: &str) -> IResult<&str, &str> {
+    let (input, _) = alt((
+        tag_no_case("read_parquet"),
+        tag_no_case("read_csv"),
+        tag_no_case("read_json"),
+    ))
+    .parse(input)?;
+    let (input, _) = multispace0(input)?;
+    let (input, _) = char('(')(input)?;
+    let (input, _) = multispace0(input)?;
+    let (input, path) = parse_s3_object_lit(input)?;
+    let (input, _) = multispace0(input)?;
+    let (input, _) = char(')')(input)?;
+    Ok((input, path))
+}
+
+/// SELECT ... FROM 's3://...'
+fn parse_s3_object_select_from(input: &str) -> IResult<&str, &str> {
+    let (input, _) = tag_no_case("FROM").parse(input)?;
+    let (input, _) = multispace0(input)?;
+    let (input, path) = parse_s3_object_lit(input)?;
+    Ok((input, path))
+}
+/// 's3://...'
+fn parse_s3_object_lit(input: &str) -> IResult<&str, &str> {
+    let (input, _) = quote(input)?;
+    let (input, _) = tag("s3://").parse(input)?;
+    let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
+    let (input, _) = quote(input)?;
+    Ok((input, path))
+}
+
+fn quote(input: &str) -> IResult<&str, char> {
+    alt((char('\''), char('\"'))).parse(input)
+}
+
+fn parse_resource_lit(input: &str) -> IResult<&str, &str> {
+    let (input, _) = quote(input)?;
+    let (input, _) = alt((tag("$res:"), tag("res://"))).parse(input)?;
+    let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?;
+    let (input, _) = quote(input)?;
+    Ok((input, path))
+}
+
+fn parse_comment(input: &str) -> IResult<&str, &str> {
+    let (input, _) = tag("--").parse(input)?;
+    let (input, comment) = take_while(|c| c != '\n')(input)?;
+    Ok((input, comment))
+}
@@ -19,6 +19,9 @@ pub use windmill_parser::{Arg, MainArgSignature, Typ};
 pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__";
 pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__";

+mod asset_parser;
+pub use asset_parser::parse_assets;
+
 pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
    let parsed = parse_mysql_file(&code)?;
    if let Some(x) = parsed {