From 2ab5345e617cbf336447768232c6f6e39cd406a3 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Fri, 18 Jul 2025 00:15:01 +0200 Subject: [PATCH] Assets refactor (#6217) * Moved logic to FlowAssetsProvider * Remove assetsMap in flow * do not parse everything on mount + only check for missing assets fields * add assets field in backend * remove fallbackAccessTypes * better structure and less queries / parsing * Fix assets not showing when pulling raw_flow from jobs * flow assets ctx for job run * Fix transitive assets fetching * Fix input args asset node * enablePathScriptAndFlowAssets flag * edit btn for variable * untrack refresh * move parseInputArgsAssets * Assets tab in runs * Update FlowStatusViewerInner to svelte 5 + fix asset sync bug * avoid toast error on bad resource * fetch res metadata for input arg asset * Job assets viewer in run page * r/w selector * remove indigo badge * store alt_access_type state in ScriptEditor * Don't parse assets in flow script editor * Add alt_access_type in backend * show Read as selected by default to avoid giving the feeling of having made a decision * keep alt_access_type when reparsing in flow raw scripts * Remove variable asset kind, and save assets for scripts * remove all backend asset parsing * R/W/RW selector button nits * fix insert into assets not saving alt access type * support named arguments in python asset parser * improve asset usage drawer R/W indicator * update legacy $res: syntax * reactivity issue * remove last variable asset stuff * sqlx prepare * tooltip explainer * deprecated variable asset nit --- ...b21270131e9e93ca10d195664e7e5a774fe9e.json | 91 +++++++ .../20250717135503_assets_refactor.down.sql | 7 + .../20250717135503_assets_refactor.up.sql | 8 + .../windmill-parser-py/src/asset_parser.rs | 36 +-- .../windmill-parser-ts/src/asset_parser.rs | 2 - .../windmill-parser/src/asset_parser.rs | 3 - backend/tests/worker.rs | 12 +- backend/windmill-api/openapi.yaml | 16 +- backend/windmill-api/src/flows.rs | 2 +- backend/windmill-api/src/scripts.rs | 24 +- backend/windmill-common/src/assets.rs | 79 +----- backend/windmill-common/src/flows.rs | 14 +- backend/windmill-common/src/scripts.rs | 8 +- .../windmill-worker/src/worker_lockfiles.rs | 13 +- frontend/src/lib/components/EditorBar.svelte | 4 +- .../src/lib/components/FlowBuilder.svelte | 13 + .../lib/components/FlowStatusViewer.svelte | 4 +- .../components/FlowStatusViewerInner.svelte | 243 +++++++++++------- .../src/lib/components/ScriptBuilder.svelte | 7 +- .../src/lib/components/ScriptEditor.svelte | 50 ++-- .../lib/components/assets/AssetButtons.svelte | 56 ++++ .../assets/AssetsDropdownButton.svelte | 141 +++++----- .../assets/AssetsUsageDrawer.svelte | 21 +- .../components/assets/JobAssetsViewer.svelte | 95 +++++++ .../src/lib/components/assets/README_DEV.md | 0 frontend/src/lib/components/assets/lib.ts | 63 ++++- .../components/flows/FlowAssetsHandler.svelte | 162 ++++++++++++ .../flows/content/FlowModuleComponent.svelte | 25 +- frontend/src/lib/components/flows/types.ts | 3 +- .../lib/components/graph/FlowGraphV2.svelte | 128 +-------- .../components/graph/graphBuilder.svelte.ts | 31 ++- .../graph/renderers/edges/BaseEdge.svelte | 15 +- .../graph/renderers/nodes/AssetNode.svelte | 71 ++--- .../nodes/AssetsOverflowedNode.svelte | 4 +- .../components/icons/AssetGenericIcon.svelte | 3 - .../lib/components/icons/AssetVarIcon.svelte | 38 --- .../src/lib/components/runs/JobPreview.svelte | 17 +- frontend/src/lib/components/script_builder.ts | 4 +- frontend/src/lib/svelte5Utils.svelte.ts | 40 +-- frontend/src/lib/utils.ts | 17 ++ .../(logged)/flows/get/[...path]/+page.svelte | 14 + .../(root)/(logged)/run/[...run]/+page.svelte | 25 +- openflow.openapi.yaml | 10 +- 43 files changed, 994 insertions(+), 625 deletions(-) create mode 100644 backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json create mode 100644 backend/migrations/20250717135503_assets_refactor.down.sql create mode 100644 backend/migrations/20250717135503_assets_refactor.up.sql create mode 100644 frontend/src/lib/components/assets/AssetButtons.svelte create mode 100644 frontend/src/lib/components/assets/JobAssetsViewer.svelte create mode 100644 frontend/src/lib/components/assets/README_DEV.md create mode 100644 frontend/src/lib/components/flows/FlowAssetsHandler.svelte delete mode 100644 frontend/src/lib/components/icons/AssetVarIcon.svelte diff --git a/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json new file mode 100644 index 0000000000..e8b39caef7 --- /dev/null +++ b/backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json @@ -0,0 +1,91 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb" + ] + } + } + }, + { + "Custom": { + "name": "script_kind", + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval", + "preprocessor" + ] + } + } + }, + "Varchar", + "Bool", + "VarcharArray", + "Int4", + "Int4", + "Int4", + "Bool", + "Bool", + "Int2", + "Bool", + "Bool", + "Int4", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Bool", + "Text", + "Bool", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e" +} diff --git a/backend/migrations/20250717135503_assets_refactor.down.sql b/backend/migrations/20250717135503_assets_refactor.down.sql new file mode 100644 index 0000000000..76780e9200 --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.down.sql @@ -0,0 +1,7 @@ +-- Add 'variable' kind back +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; + +ALTER TABLE script DROP COLUMN assets; diff --git a/backend/migrations/20250717135503_assets_refactor.up.sql b/backend/migrations/20250717135503_assets_refactor.up.sql new file mode 100644 index 0000000000..bd17555a9c --- /dev/null +++ b/backend/migrations/20250717135503_assets_refactor.up.sql @@ -0,0 +1,8 @@ +ALTER TABLE script ADD COLUMN assets jsonb; + +-- Remove 'variable' kind +DELETE FROM asset WHERE kind = 'variable'; +ALTER TABLE asset ALTER column kind TYPE VARCHAR; +DROP TYPE asset_kind; +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource'); +ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND; \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index d6f16824c7..a96057af1e 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -57,27 +57,27 @@ impl AssetsFinder { }) .ok_or(())?; + use AssetKind::*; let (kind, access_type, arg) = match ident.as_str() { - "load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)), - "write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)), - "get_resource" => (AssetKind::Resource, None, Arg::Pos(0)), - "set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")), - "get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)), - "get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)), - "set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)), + "load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")), + "write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")), + "get_resource" => (Resource, None, Arg::Pos(0, "path")), + "set_resource" => (Resource, Some(W), Arg::Pos(0, "path")), + "get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), + "get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")), _ => return Err(()), }; let arg_val = match arg { - Arg::Pos(i) => node.args.get(i), - Arg::Named(name) => node - .keywords - .iter() - .find(|kw| kw.arg.as_deref() == Some(name)) - .map(|kw| &kw.value), + Arg::Pos(i, name) => node.args.get(i).or_else(|| { + // Get arg by name + node.keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .map(|kw| &kw.value) + }), }; match arg_val { @@ -93,6 +93,6 @@ impl AssetsFinder { } enum Arg { - Pos(usize), - Named(&'static str), + // Positional arguments in python can also be used by their name + Pos(usize, &'static str), } diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index dcd6681f5f..6b81272ec1 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -84,8 +84,6 @@ impl AssetsFinder { "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), "polarsConnectionSettings" => (AssetKind::Resource, None, 0), - "getVariable" => (AssetKind::Variable, Some(R), 0), - "setVariable" => (AssetKind::Variable, Some(W), 0), _ => return Err(()), }; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 893843342b..6857e8d68e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -15,7 +15,6 @@ use AssetUsageAccessType::*; pub enum AssetKind { S3Object, Resource, - Variable, } #[derive(Serialize)] @@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[6..])) } else if s.starts_with("$res:") { Some((AssetKind::Resource, &s[5..])) - } else if s.starts_with("var://") { - Some((AssetKind::Variable, &s[6..])) } else { None } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 28bce55024..65263152dc 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1147,7 +1147,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1191,7 +1191,7 @@ async fn test_deno_flow(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, } .into(), stop_after_if: Default::default(), @@ -1321,7 +1321,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1376,7 +1376,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), @@ -1416,7 +1416,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), @@ -1482,7 +1482,7 @@ async fn test_deno_flow_same_worker(db: Pool) { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }.into(), stop_after_if: Default::default(), stop_after_all_iters_if: Default::default(), diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 777c708c93..e2259a4900 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3115,7 +3115,7 @@ paths: type: array items: $ref: "#/components/schemas/ContextualVariable" - + /w/{workspace}/workspaces/get_secondary_storage_names: get: summary: get secondary storage names @@ -13838,20 +13838,27 @@ components: type: boolean on_behalf_of_email: type: string - fallback_access_types: + assets: type: array items: type: object - required: [path, kind, access_type] + required: + - path + - kind properties: path: type: string kind: type: string - enum: [s3object, resource] + enum: + - s3object + - resource access_type: type: string enum: [r, w, rw] + alt_access_type: + type: string + enum: [r, w, rw] required: - path @@ -17606,7 +17613,6 @@ components: enum: - s3object - resource - - variable Asset: type: object properties: diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 4ebeecb675..f9d7827341 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1391,7 +1391,7 @@ mod tests { concurrent_limit: None, concurrency_time_window_s: None, is_trigger: None, - asset_fallback_access_types: None, + assets: None, }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 98c65c1656..e32a2da82b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -43,7 +43,7 @@ use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; use windmill_common::{ - assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind}, + assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, worker::CLOUD_HOSTED, }; @@ -116,6 +116,9 @@ pub struct ScriptWDraft { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } pub fn global_service() -> Router { @@ -771,8 +774,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)", &w_id, &hash.0, ns.path, @@ -810,6 +813,7 @@ async fn create_script_internal<'c>( None }, validate_schema, + ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()) ) .execute(&mut *tx) .await?; @@ -924,16 +928,8 @@ async fn create_script_internal<'c>( } clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() { - insert_asset_usage( - &mut *tx, - &w_id, - asset, - ns.fallback_access_types.as_ref().map(Vec::as_slice), - &ns.path, - AssetUsageKind::Script, - ) - .await?; + for asset in ns.assets.as_ref().into_iter().flatten() { + insert_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script).await?; } let permissioned_as = username_to_permissioned_as(&authed.username); @@ -1126,7 +1122,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 6b720b368d..20c711c127 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,8 +1,7 @@ use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; -use windmill_parser::asset_parser::ParseAssetsResult; -use crate::{error, scripts::ScriptLang}; +use crate::error; #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] #[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] @@ -10,7 +9,8 @@ use crate::{error, scripts::ScriptLang}; pub enum AssetKind { S3Object, Resource, - Variable, + // Avoid unnexpected crashes when deserializing old assets + Variable, // Deprecated } #[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] @@ -41,85 +41,28 @@ pub struct AssetUsage { pub access_type: AssetUsageAccessType, } -#[derive(Serialize, Deserialize, Debug, Clone, Hash)] -pub struct AssetWithAccessType { +#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)] +pub struct AssetWithAltAccessType { pub path: String, pub kind: AssetKind, - pub access_type: AssetUsageAccessType, -} - -pub fn parse_assets( - input: &str, - lang: ScriptLang, -) -> anyhow::Result>>> { - let r = match lang { - ScriptLang::Python3 => windmill_parser_py::parse_assets(input), - ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| { - a.iter() - .map(|a| ParseAssetsResult { - path: a.path.to_string(), - access_type: a.access_type, - kind: a.kind, - }) - .collect() - }), - ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => { - windmill_parser_ts::parse_assets(input) - } - _ => return Ok(None), - }; - return r.map(Some); -} - -impl From for AssetKind { - fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self { - match kind { - windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, - windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, - windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable, - } - } -} -impl From for AssetUsageAccessType { - fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self { - match access_type { - windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R, - windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W, - windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW, - } - } + pub access_type: Option, + pub alt_access_type: Option, } pub async fn insert_asset_usage<'e>( executor: impl PgExecutor<'e>, workspace_id: &str, - parsed_asset: &ParseAssetsResult, - fallback_access_types: Option<&[AssetWithAccessType]>, + asset: &AssetWithAltAccessType, usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { - let kind: AssetKind = parsed_asset.kind.into(); - let asset_alternative_access_type = || { - fallback_access_types - .as_ref() - .and_then(|v| { - v.iter() - .find(|a| a.kind == kind && a.path == parsed_asset.path) - }) - .map(|a| a.access_type) - }; - let access_type: Option = parsed_asset - .access_type - .map(Into::into) - .or_else(asset_alternative_access_type); - sqlx::query!( r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#, workspace_id, - parsed_asset.path, - kind as AssetKind, - access_type as Option, + asset.path, + asset.kind as AssetKind, + (asset.access_type.or(asset.alt_access_type)) as Option, usage_path, usage_kind as AssetUsageKind ) diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index b6d8f422e2..4a4cb200a8 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -18,7 +18,7 @@ use sqlx::types::Json; use sqlx::types::JsonRawValue; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, cache, error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, @@ -506,7 +506,7 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "Option::is_none")] - asset_fallback_access_types: Option>, + assets: Option>, }, Identity, // Internal only, never exposed to the frontend. @@ -526,6 +526,8 @@ pub enum FlowModuleValue { concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, + #[serde(skip_serializing_if = "Option::is_none")] + assets: Option>, }, } @@ -560,7 +562,7 @@ struct UntaggedFlowModuleValue { id: Option, default_node: Option, modules_node: Option, - asset_fallback_access_types: Option>, + assets: Option>, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -635,7 +637,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, - asset_fallback_access_types: untagged.asset_fallback_access_types, + assets: untagged.assets, }), "flowscript" => Ok(FlowModuleValue::FlowScript { input_transforms: untagged.input_transforms.unwrap_or_default(), @@ -650,6 +652,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { concurrent_limit: untagged.concurrent_limit, concurrency_time_window_s: untagged.concurrency_time_window_s, is_trigger: untagged.is_trigger, + assets: untagged.assets, }), "identity" => Ok(FlowModuleValue::Identity), other => Err(serde::de::Error::unknown_variant( @@ -785,6 +788,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, } = std::mem::replace(&mut val, Identity) else { unreachable!() @@ -808,7 +812,7 @@ pub async fn resolve_module( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types: None, + assets, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 57dd7d2498..ff2f1865af 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -13,7 +13,7 @@ use std::{ }; use crate::{ - assets::AssetWithAccessType, + assets::AssetWithAltAccessType, error::{to_anyhow, Error}, utils::http_get_from_hub, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, @@ -250,6 +250,9 @@ pub struct Script { pub has_preprocessor: Option, #[serde(skip_serializing_if = "Option::is_none")] pub on_behalf_of_email: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[sqlx(json(nullable))] + pub assets: Option>, } #[derive(Serialize, sqlx::FromRow)] @@ -349,7 +352,8 @@ pub struct NewScript { pub codebase: Option, pub has_preprocessor: Option, pub on_behalf_of_email: Option, - pub fallback_access_types: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub assets: Option>, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index ffd6909032..18720456c5 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -11,9 +11,7 @@ use serde_json::{json, Value}; use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; -use windmill_common::assets::{ - clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind, -}; +use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind}; use windmill_common::error::Error; use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; @@ -946,7 +944,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, } = e.get_value()? else { match e.get_value()? { @@ -1139,12 +1137,11 @@ async fn lock_modules<'c>( continue; }; - for asset in parse_assets(&content, language)?.iter().flatten() { + for asset in assets.iter().flatten() { insert_asset_usage( &mut *tx, &job.workspace_id, asset, - asset_fallback_access_types.as_ref().map(Vec::as_slice), job_path, AssetUsageKind::Flow, ) @@ -1266,7 +1263,7 @@ async fn lock_modules<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, - asset_fallback_access_types, + assets, }); new_flow_modules.push(e); @@ -1418,6 +1415,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, .. } = std::mem::replace(&mut val, Identity) else { @@ -1436,6 +1434,7 @@ async fn reduce_flow<'c>( concurrent_limit, concurrency_time_window_s, is_trigger, + assets, }; } ForloopFlow { modules, modules_node, .. } diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 41dc59c0f2..749f8d163d 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -593,10 +593,10 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType] if (!t) { sendUserToast(`Resource type ${resType} is not supported in DuckDB`, true) - editor.insertAtCursor(`'$res:${path}'`) + editor.insertAtCursor(`'res://${path}'`) return } else { - editor.insertAtCursor(`ATTACH '$res:${path}' AS db (TYPE ${t});`) + editor.insertAtCursor(`ATTACH 'res://${path}' AS db (TYPE ${t});`) } } diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 2672b491de..da9fb506ca 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -86,6 +86,7 @@ } from './stepHistoryLoader.svelte' import type { FlowBuilderProps } from './flow_builder' import { ModulesTestStates } from './modulesTest.svelte' + import FlowAssetsHandler, { initFlowGraphAssetsCtx } from './flows/FlowAssetsHandler.svelte' let { initialPath = $bindable(''), @@ -609,6 +610,11 @@ outputPickerOpenFns }) + setContext( + 'FlowGraphAssetContext', + initFlowGraphAssetsCtx({ getModules: () => flowStore.val.value.modules }) + ) + // Add triggers context store const triggersState = $state( new Triggers( @@ -1256,3 +1262,10 @@ renderCount += 1 }} /> + + diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 9afcd99f85..305f9d3c20 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -74,8 +74,8 @@ }} globalModuleStates={[]} globalDurationStatuses={[]} - bind:localModuleStates - bind:localDurationStatuses + {localModuleStates} + {localDurationStatuses} bind:selectedNode={selectedJobStep} on:start on:done diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index dd8f70eb17..0060fc1ade 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -1,4 +1,6 @@ {#if notAnonynmous} @@ -906,7 +950,7 @@

For performance reasons, only the last 20 items are shown by default