starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-08 16:03:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Assets refactor (#6217)

Browse Source 
* Moved logic to FlowAssetsProvider

* Remove assetsMap in flow

* do not parse everything on mount + only check for missing assets fields

* add assets field in backend

* remove fallbackAccessTypes

* better structure and less queries / parsing

* Fix assets not showing when pulling raw_flow from jobs

* flow assets ctx for job run

* Fix transitive assets fetching

* Fix input args asset node

* enablePathScriptAndFlowAssets flag

* edit btn for variable

* untrack refresh

* move parseInputArgsAssets

* Assets tab in runs

* Update FlowStatusViewerInner to svelte 5 + fix asset sync bug

* avoid toast error on bad resource

* fetch res metadata for input arg asset

* Job assets viewer in run page

* r/w selector

* remove indigo badge

* store alt_access_type state in ScriptEditor

* Don't parse assets in flow script editor

* Add alt_access_type in backend

* show Read as selected by default to avoid giving the feeling of having made a decision

* keep alt_access_type when reparsing in flow raw scripts

* Remove variable asset kind, and save assets for scripts

* remove all backend asset parsing

* R/W/RW selector button nits

* fix insert into assets not saving alt access type

* support named arguments in python asset parser

* improve asset usage drawer R/W indicator

* update legacy $res: syntax

* reactivity issue

* remove last variable asset stuff

* sqlx prepare

* tooltip explainer

* deprecated variable asset nit
This commit is contained in:
Diego Imbert
2025-07-17 22:15:01 +00:00
committed by GitHub
parent ec1ed0ba6b
commit 2ab5345e61
43 changed files with 994 additions and 625 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e.json
Generated
+91
View File
@@ -0,0 +1,91 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb"
]
}
}
},
{
"Custom": {
"name": "script_kind",
"kind": {
"Enum": [
"script",
"trigger",
"failure",
"command",
"approval",
"preprocessor"
]
}
}
},
"Varchar",
"Bool",
"VarcharArray",
"Int4",
"Int4",
"Int4",
"Bool",
"Bool",
"Int2",
"Bool",
"Bool",
"Int4",
"Varchar",
"Bool",
"Bool",
"Varchar",
"Bool",
"Text",
"Bool",
"Jsonb"
]
},
"nullable": []
},
"hash": "1a54356c1e1353950bf6ab1d25ab21270131e9e93ca10d195664e7e5a774fe9e"
}
backend/migrations/20250717135503_assets_refactor.down.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add 'variable' kind back
ALTER TABLE asset ALTER column kind TYPE VARCHAR;
DROP TYPE asset_kind;
CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource', 'variable');
ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;
ALTER TABLE script DROP COLUMN assets;
backend/migrations/20250717135503_assets_refactor.up.sql
+8
View File
@@ -0,0 +1,8 @@
ALTER TABLE script ADD COLUMN assets jsonb;
-- Remove 'variable' kind
DELETE FROM asset WHERE kind = 'variable';
ALTER TABLE asset ALTER column kind TYPE VARCHAR;
DROP TYPE asset_kind;
CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource');
ALTER TABLE asset ALTER column kind TYPE ASSET_KIND using kind::ASSET_KIND;
backend/parsers/windmill-parser-py/src/asset_parser.rs
+18 -18
View File
@@ -57,27 +57,27 @@ impl AssetsFinder {
})
.ok_or(())?;
use AssetKind::*;
let (kind, access_type, arg) = match ident.as_str() {
"load_s3_file" => (AssetKind::S3Object, Some(R), Arg::Pos(0)),
"load_s3_file_reader" => (AssetKind::S3Object, Some(R), Arg::Pos(0)),
"write_s3_file" => (AssetKind::S3Object, Some(W), Arg::Pos(0)),
"get_resource" => (AssetKind::Resource, None, Arg::Pos(0)),
"set_resource" => (AssetKind::Resource, Some(W), Arg::Named("path")),
"get_boto3_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_polars_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_duckdb_connection_settings" => (AssetKind::Resource, None, Arg::Pos(0)),
"get_variable" => (AssetKind::Variable, Some(R), Arg::Pos(0)),
"set_variable" => (AssetKind::Variable, Some(W), Arg::Pos(0)),
"load_s3_file" => (S3Object, Some(R), Arg::Pos(0, "s3object")),
"load_s3_file_reader" => (S3Object, Some(R), Arg::Pos(0, "s3object")),
"write_s3_file" => (S3Object, Some(W), Arg::Pos(0, "s3object")),
"get_resource" => (Resource, None, Arg::Pos(0, "path")),
"set_resource" => (Resource, Some(W), Arg::Pos(0, "path")),
"get_boto3_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
"get_polars_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
"get_duckdb_connection_settings" => (Resource, None, Arg::Pos(0, "s3_resource_path")),
_ => return Err(()),
};
let arg_val = match arg {
Arg::Pos(i) => node.args.get(i),
Arg::Named(name) => node
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value),
Arg::Pos(i, name) => node.args.get(i).or_else(|| {
// Get arg by name
node.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value)
}),
};
match arg_val {
@@ -93,6 +93,6 @@ impl AssetsFinder {
}
enum Arg {
Pos(usize),
Named(&'static str),
// Positional arguments in python can also be used by their name
Pos(usize, &'static str),
}
backend/parsers/windmill-parser-ts/src/asset_parser.rs
-2
View File
@@ -84,8 +84,6 @@ impl AssetsFinder {
"denoS3LightClientSettings" => (AssetKind::Resource, None, 0),
"duckdbConnectionSettings" => (AssetKind::Resource, None, 0),
"polarsConnectionSettings" => (AssetKind::Resource, None, 0),
"getVariable" => (AssetKind::Variable, Some(R), 0),
"setVariable" => (AssetKind::Variable, Some(W), 0),
_ => return Err(()),
};
backend/parsers/windmill-parser/src/asset_parser.rs
-3
View File
@@ -15,7 +15,6 @@ use AssetUsageAccessType::*;
pub enum AssetKind {
S3Object,
Resource,
Variable,
}
#[derive(Serialize)]
@@ -57,8 +56,6 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
Some((AssetKind::Resource, &s[6..]))
} else if s.starts_with("$res:") {
Some((AssetKind::Resource, &s[5..]))
} else if s.starts_with("var://") {
Some((AssetKind::Variable, &s[6..]))
} else {
None
}
backend/tests/worker.rs
+6 -6
View File
@@ -1147,7 +1147,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}
.into(),
stop_after_if: Default::default(),
@@ -1191,7 +1191,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}
.into(),
stop_after_if: Default::default(),
@@ -1321,7 +1321,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
@@ -1376,7 +1376,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
@@ -1416,7 +1416,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
@@ -1482,7 +1482,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}.into(),
stop_after_if: Default::default(),
stop_after_all_iters_if: Default::default(),
backend/windmill-api/openapi.yaml
+11 -5
View File
@@ -3115,7 +3115,7 @@ paths:
type: array
items:
$ref: "#/components/schemas/ContextualVariable"
/w/{workspace}/workspaces/get_secondary_storage_names:
get:
summary: get secondary storage names
@@ -13838,20 +13838,27 @@ components:
type: boolean
on_behalf_of_email:
type: string
fallback_access_types:
assets:
type: array
items:
type: object
required: [path, kind, access_type]
required:
- path
- kind
properties:
path:
type: string
kind:
type: string
enum: [s3object, resource]
enum:
- s3object
- resource
access_type:
type: string
enum: [r, w, rw]
alt_access_type:
type: string
enum: [r, w, rw]
required:
- path
@@ -17606,7 +17613,6 @@ components:
enum:
- s3object
- resource
- variable
Asset:
type: object
properties:
backend/windmill-api/src/flows.rs
+1 -1
View File
@@ -1391,7 +1391,7 @@ mod tests {
concurrent_limit: None,
concurrency_time_window_s: None,
is_trigger: None,
asset_fallback_access_types: None,
assets: None,
}),
stop_after_if: Some(StopAfterIf {
expr: "foo = 'bar'".to_string(),
backend/windmill-api/src/scripts.rs
+10 -14
View File
@@ -43,7 +43,7 @@ use windmill_audit::ActionKind;
use windmill_worker::process_relative_imports;
use windmill_common::{
assets::{clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind},
assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType},
error::to_anyhow,
worker::CLOUD_HOSTED,
};
@@ -116,6 +116,9 @@ pub struct ScriptWDraft {
pub has_preprocessor: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_behalf_of_email: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(json(nullable))]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
pub fn global_service() -> Router {
@@ -771,8 +774,8 @@ async fn create_script_internal<'c>(
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33)",
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34)",
&w_id,
&hash.0,
ns.path,
@@ -810,6 +813,7 @@ async fn create_script_internal<'c>(
None
},
validate_schema,
ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok())
)
.execute(&mut *tx)
.await?;
@@ -924,16 +928,8 @@ async fn create_script_internal<'c>(
}
clear_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?;
for asset in parse_assets(&ns.content, ns.language)?.iter().flatten() {
insert_asset_usage(
&mut *tx,
&w_id,
asset,
ns.fallback_access_types.as_ref().map(Vec::as_slice),
&ns.path,
AssetUsageKind::Script,
)
.await?;
for asset in ns.assets.as_ref().into_iter().flatten() {
insert_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script).await?;
}
let permissioned_as = username_to_permissioned_as(&authed.username);
@@ -1126,7 +1122,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2
ORDER BY script.created_at DESC LIMIT 1",
backend/windmill-common/src/assets.rs
+11 -68
View File
@@ -1,8 +1,7 @@
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
use windmill_parser::asset_parser::ParseAssetsResult;
use crate::{error, scripts::ScriptLang};
use crate::error;
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]
#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")]
@@ -10,7 +9,8 @@ use crate::{error, scripts::ScriptLang};
pub enum AssetKind {
S3Object,
Resource,
Variable,
// Avoid unnexpected crashes when deserializing old assets
Variable, // Deprecated
}
#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)]
@@ -41,85 +41,28 @@ pub struct AssetUsage {
pub access_type: AssetUsageAccessType,
}
#[derive(Serialize, Deserialize, Debug, Clone, Hash)]
pub struct AssetWithAccessType {
#[derive(Serialize, Deserialize, Debug, Clone, Hash, sqlx::Type)]
pub struct AssetWithAltAccessType {
pub path: String,
pub kind: AssetKind,
pub access_type: AssetUsageAccessType,
}
pub fn parse_assets(
input: &str,
lang: ScriptLang,
) -> anyhow::Result<Option<Vec<ParseAssetsResult<String>>>> {
let r = match lang {
ScriptLang::Python3 => windmill_parser_py::parse_assets(input),
ScriptLang::DuckDb => windmill_parser_sql::parse_assets(input).map(|a| {
a.iter()
.map(|a| ParseAssetsResult {
path: a.path.to_string(),
access_type: a.access_type,
kind: a.kind,
})
.collect()
}),
ScriptLang::Deno | ScriptLang::Bun | ScriptLang::Nativets => {
windmill_parser_ts::parse_assets(input)
}
_ => return Ok(None),
};
return r.map(Some);
}
impl From<windmill_parser::asset_parser::AssetKind> for AssetKind {
fn from(kind: windmill_parser::asset_parser::AssetKind) -> Self {
match kind {
windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object,
windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource,
windmill_parser::asset_parser::AssetKind::Variable => AssetKind::Variable,
}
}
}
impl From<windmill_parser::asset_parser::AssetUsageAccessType> for AssetUsageAccessType {
fn from(access_type: windmill_parser::asset_parser::AssetUsageAccessType) -> Self {
match access_type {
windmill_parser::asset_parser::AssetUsageAccessType::R => AssetUsageAccessType::R,
windmill_parser::asset_parser::AssetUsageAccessType::W => AssetUsageAccessType::W,
windmill_parser::asset_parser::AssetUsageAccessType::RW => AssetUsageAccessType::RW,
}
}
pub access_type: Option<AssetUsageAccessType>,
pub alt_access_type: Option<AssetUsageAccessType>,
}
pub async fn insert_asset_usage<'e>(
executor: impl PgExecutor<'e>,
workspace_id: &str,
parsed_asset: &ParseAssetsResult<String>,
fallback_access_types: Option<&[AssetWithAccessType]>,
asset: &AssetWithAltAccessType,
usage_path: &str,
usage_kind: AssetUsageKind,
) -> error::Result<()> {
let kind: AssetKind = parsed_asset.kind.into();
let asset_alternative_access_type = || {
fallback_access_types
.as_ref()
.and_then(|v| {
v.iter()
.find(|a| a.kind == kind && a.path == parsed_asset.path)
})
.map(|a| a.access_type)
};
let access_type: Option<AssetUsageAccessType> = parsed_asset
.access_type
.map(Into::into)
.or_else(asset_alternative_access_type);
sqlx::query!(
r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind)
VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING"#,
workspace_id,
parsed_asset.path,
kind as AssetKind,
access_type as Option<AssetUsageAccessType>,
asset.path,
asset.kind as AssetKind,
(asset.access_type.or(asset.alt_access_type)) as Option<AssetUsageAccessType>,
usage_path,
usage_kind as AssetUsageKind
)
backend/windmill-common/src/flows.rs
+9 -5
View File
@@ -18,7 +18,7 @@ use sqlx::types::Json;
use sqlx::types::JsonRawValue;
use crate::{
assets::AssetWithAccessType,
assets::AssetWithAltAccessType,
cache,
error::Error,
more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
@@ -506,7 +506,7 @@ pub enum FlowModuleValue {
#[serde(skip_serializing_if = "Option::is_none")]
is_trigger: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
asset_fallback_access_types: Option<Vec<AssetWithAccessType>>,
assets: Option<Vec<AssetWithAltAccessType>>,
},
Identity,
// Internal only, never exposed to the frontend.
@@ -526,6 +526,8 @@ pub enum FlowModuleValue {
concurrency_time_window_s: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
is_trigger: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
assets: Option<Vec<AssetWithAltAccessType>>,
},
}
@@ -560,7 +562,7 @@ struct UntaggedFlowModuleValue {
id: Option<FlowNodeId>,
default_node: Option<FlowNodeId>,
modules_node: Option<FlowNodeId>,
asset_fallback_access_types: Option<Vec<AssetWithAccessType>>,
assets: Option<Vec<AssetWithAltAccessType>>,
}
impl<'de> Deserialize<'de> for FlowModuleValue {
@@ -635,7 +637,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
concurrent_limit: untagged.concurrent_limit,
concurrency_time_window_s: untagged.concurrency_time_window_s,
is_trigger: untagged.is_trigger,
asset_fallback_access_types: untagged.asset_fallback_access_types,
assets: untagged.assets,
}),
"flowscript" => Ok(FlowModuleValue::FlowScript {
input_transforms: untagged.input_transforms.unwrap_or_default(),
@@ -650,6 +652,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
concurrent_limit: untagged.concurrent_limit,
concurrency_time_window_s: untagged.concurrency_time_window_s,
is_trigger: untagged.is_trigger,
assets: untagged.assets,
}),
"identity" => Ok(FlowModuleValue::Identity),
other => Err(serde::de::Error::unknown_variant(
@@ -785,6 +788,7 @@ pub async fn resolve_module(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
@@ -808,7 +812,7 @@ pub async fn resolve_module(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types: None,
assets,
};
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
backend/windmill-common/src/scripts.rs
+6 -2
View File
@@ -13,7 +13,7 @@ use std::{
};
use crate::{
assets::AssetWithAccessType,
assets::AssetWithAltAccessType,
error::{to_anyhow, Error},
utils::http_get_from_hub,
DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL,
@@ -250,6 +250,9 @@ pub struct Script {
pub has_preprocessor: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub on_behalf_of_email: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[sqlx(json(nullable))]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
#[derive(Serialize, sqlx::FromRow)]
@@ -349,7 +352,8 @@ pub struct NewScript {
pub codebase: Option<String>,
pub has_preprocessor: Option<bool>,
pub on_behalf_of_email: Option<String>,
pub fallback_access_types: Option<Vec<AssetWithAccessType>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub assets: Option<Vec<AssetWithAltAccessType>>,
}
fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
backend/windmill-worker/src/worker_lockfiles.rs
+6 -7
View File
@@ -11,9 +11,7 @@ use serde_json::{json, Value};
use sha2::Digest;
use sqlx::types::Json;
use uuid::Uuid;
use windmill_common::assets::{
clear_asset_usage, insert_asset_usage, parse_assets, AssetUsageKind,
};
use windmill_common::assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind};
use windmill_common::error::Error;
use windmill_common::error::Result;
use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId};
@@ -946,7 +944,7 @@ async fn lock_modules<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types,
assets,
} = e.get_value()?
else {
match e.get_value()? {
@@ -1139,12 +1137,11 @@ async fn lock_modules<'c>(
continue;
};
for asset in parse_assets(&content, language)?.iter().flatten() {
for asset in assets.iter().flatten() {
insert_asset_usage(
&mut *tx,
&job.workspace_id,
asset,
asset_fallback_access_types.as_ref().map(Vec::as_slice),
job_path,
AssetUsageKind::Flow,
)
@@ -1266,7 +1263,7 @@ async fn lock_modules<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
asset_fallback_access_types,
assets,
});
new_flow_modules.push(e);
@@ -1418,6 +1415,7 @@ async fn reduce_flow<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
..
} = std::mem::replace(&mut val, Identity)
else {
@@ -1436,6 +1434,7 @@ async fn reduce_flow<'c>(
concurrent_limit,
concurrency_time_window_s,
is_trigger,
assets,
};
}
ForloopFlow { modules, modules_node, .. }
frontend/src/lib/components/EditorBar.svelte
+2 -2
View File
@@ -593,10 +593,10 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS
let t = { postgresql: 'postgres', mysql: 'mysql', bigquery: 'bigquery' }[resType]
if (!t) {
sendUserToast(`Resource type ${resType} is not supported in DuckDB`, true)
editor.insertAtCursor(`'$res:${path}'`)
editor.insertAtCursor(`'res://${path}'`)
return
} else {
editor.insertAtCursor(`ATTACH '$res:${path}' AS db (TYPE ${t});`)
editor.insertAtCursor(`ATTACH 'res://${path}' AS db (TYPE ${t});`)
}
}
frontend/src/lib/components/FlowBuilder.svelte
+13
View File
@@ -86,6 +86,7 @@
} from './stepHistoryLoader.svelte'
import type { FlowBuilderProps } from './flow_builder'
import { ModulesTestStates } from './modulesTest.svelte'
import FlowAssetsHandler, { initFlowGraphAssetsCtx } from './flows/FlowAssetsHandler.svelte'
let {
initialPath = $bindable(''),
@@ -609,6 +610,11 @@
outputPickerOpenFns
})
setContext(
'FlowGraphAssetContext',
initFlowGraphAssetsCtx({ getModules: () => flowStore.val.value.modules })
)
// Add triggers context store
const triggersState = $state(
new Triggers(
@@ -1256,3 +1262,10 @@
renderCount += 1
}}
/>
<FlowAssetsHandler
modules={flowStore.val.value.modules}
enableParser
enableDbExplore
enablePathScriptAndFlowAssets
/>
frontend/src/lib/components/FlowStatusViewer.svelte
+2 -2
View File
@@ -74,8 +74,8 @@
}}
globalModuleStates={[]}
globalDurationStatuses={[]}
bind:localModuleStates
bind:localDurationStatuses
{localModuleStates}
{localDurationStatuses}
bind:selectedNode={selectedJobStep}
on:start
on:done
frontend/src/lib/components/FlowStatusViewerInner.svelte
+143 -100
View File
@@ -1,4 +1,6 @@
<script lang="ts">
import FlowStatusViewerInner from './FlowStatusViewerInner.svelte'
import {
type FlowStatusModule,
type Job,
@@ -6,20 +8,20 @@
type FlowStatus,
type FlowModuleValue,
type FlowModule,
type ScriptArgs
ResourceService
} from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { base } from '$lib/base'
import FlowJobResult from './FlowJobResult.svelte'
import DisplayResult from './DisplayResult.svelte'
import { createEventDispatcher, getContext, tick } from 'svelte'
import { createEventDispatcher, getContext, setContext, tick, untrack } from 'svelte'
import { onDestroy } from 'svelte'
import { Badge, Button, Skeleton, Tab } from './common'
import Tabs from './common/tabs/Tabs.svelte'
import { type DurationStatus, type FlowStatusViewerContext, type GraphModuleState } from './graph'
import ModuleStatus from './ModuleStatus.svelte'
import { isScriptPreview, msToSec, truncateRev } from '$lib/utils'
import { clone, isScriptPreview, msToSec, readFieldsRecursively, truncateRev } from '$lib/utils'
import JobArgs from './JobArgs.svelte'
import { ChevronDown, Hourglass } from 'lucide-svelte'
import { deepEqual } from 'fast-equals'
@@ -30,8 +32,10 @@
import FlowGraphViewerStep from './FlowGraphViewerStep.svelte'
import FlowGraphV2 from './graph/FlowGraphV2.svelte'
import { buildPrefix } from './graph/graphBuilder.svelte'
import { parseAssetFromString, type AssetWithAccessType } from './assets/lib'
import { parseInputArgsAssets } from './assets/lib'
import FlowPreviewResult from './FlowPreviewResult.svelte'
import type { FlowGraphAssetContext } from './flows/types'
import { createState } from '$lib/svelte5Utils.svelte'
const dispatch = createEventDispatcher()
@@ -46,77 +50,110 @@
hideJobId
} = getContext<FlowStatusViewerContext>('FlowStatusViewer')
export let jobId: string
export let initialJob: Job | undefined = undefined
export let workspaceId: string | undefined = undefined
export let flowJobIds:
| {
moduleId: string
flowJobs: string[]
flowJobsSuccess: (boolean | undefined)[]
length: number
branchall?: boolean
}
| undefined = undefined
//only useful when forloops are optimized and the job doesn't contain the mod id anymore
export let innerModule: FlowModuleValue | undefined = undefined
export let globalRefreshes: Record<string, (clear, root) => Promise<void>> = {}
export let render = true
export let isOwner = false
export let selectedNode: string | undefined = undefined
export let globalModuleStates: Writable<Record<string, GraphModuleState>>[]
export let globalDurationStatuses: Writable<Record<string, DurationStatus>>[]
export let childFlow: boolean = false
export let isSubflow: boolean = false
export let reducedPolling = false
export let wideResults = false
export let hideFlowResult = false
export let workspace: string | undefined = $workspaceStore
export let prefix: string | undefined = undefined
export let subflowParentsGlobalModuleStates: Writable<Record<string, GraphModuleState>>[] = []
export let subflowParentsDurationStatuses: Writable<Record<string, DurationStatus>>[] = []
export let isForloopSelected = false
export let parentRecursiveRefresh: Record<string, (clear, root) => Promise<void>> = {}
export let job: Job | undefined = undefined
export let rightColumnSelect: 'timeline' | 'node_status' | 'node_definition' | 'user_states' =
'timeline'
export let localModuleStates: Writable<Record<string, GraphModuleState>> = writable({})
export let localDurationStatuses: Writable<Record<string, DurationStatus>> = writable({})
let recursiveRefresh: Record<string, (clear, root) => Promise<void>> = {}
$: inputAssets = parseInputAssets(job?.args ?? {})
function parseInputAssets(args: ScriptArgs): AssetWithAccessType[] {
const arr: AssetWithAccessType[] = []
for (const v of Object.values(args)) {
if (typeof v === 'string') {
const asset = parseAssetFromString(v)
if (asset) arr.push(asset)
} else if (v && typeof v === 'object' && typeof v['s3'] === 'string') {
const s3 = v['s3']
const storage = typeof v['storage'] == 'string' ? v['storage'] : undefined
arr.push({ kind: 's3object', path: `${storage ?? ''}/${s3}` })
}
}
return arr
interface Props {
jobId: string
initialJob?: Job | undefined
workspaceId?: string | undefined
flowJobIds?:
| {
moduleId: string
flowJobs: string[]
flowJobsSuccess: (boolean | undefined)[]
length: number
branchall?: boolean
}
| undefined
//only useful when forloops are optimized and the job doesn't contain the mod id anymore
innerModule?: FlowModuleValue | undefined
globalRefreshes?: Record<string, (clear, root) => Promise<void>>
render?: boolean
isOwner?: boolean
selectedNode?: string | undefined
globalModuleStates: Writable<Record<string, GraphModuleState>>[]
globalDurationStatuses: Writable<Record<string, DurationStatus>>[]
childFlow?: boolean
isSubflow?: boolean
reducedPolling?: boolean
wideResults?: boolean
hideFlowResult?: boolean
workspace?: string | undefined
prefix?: string | undefined
subflowParentsGlobalModuleStates?: Writable<Record<string, GraphModuleState>>[]
subflowParentsDurationStatuses?: Writable<Record<string, DurationStatus>>[]
isForloopSelected?: boolean
parentRecursiveRefresh?: Record<string, (clear, root) => Promise<void>>
job?: Job | undefined
rightColumnSelect?: 'timeline' | 'node_status' | 'node_definition' | 'user_states'
localModuleStates?: Writable<Record<string, GraphModuleState>>
localDurationStatuses?: Writable<Record<string, DurationStatus>>
}
let jobResults: any[] =
flowJobIds?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? []
let {
jobId,
initialJob = undefined,
workspaceId = undefined,
flowJobIds = undefined,
innerModule = undefined,
globalRefreshes = $bindable({}),
render = true,
isOwner = false,
selectedNode = $bindable(undefined),
globalModuleStates,
globalDurationStatuses,
childFlow = false,
isSubflow = false,
reducedPolling = false,
wideResults = false,
hideFlowResult = false,
workspace = $workspaceStore,
prefix = undefined,
subflowParentsGlobalModuleStates = [],
subflowParentsDurationStatuses = [],
isForloopSelected = false,
parentRecursiveRefresh = $bindable({}),
job = $bindable(undefined),
rightColumnSelect = $bindable('timeline'),
localModuleStates = writable({}),
localDurationStatuses = writable({})
}: Props = $props()
let recursiveRefresh: Record<string, (clear, root) => Promise<void>> = $state({})
let retry_selected = ''
// Add support for the input args assets shown as an asset node
const _flowGraphAssetsCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
let extendedFlowGraphAssetsCtx = $state(createState(clone(_flowGraphAssetsCtx)))
setContext('FlowGraphAssetContext', extendedFlowGraphAssetsCtx)
$effect(() => {
readFieldsRecursively(_flowGraphAssetsCtx)
job?.args
untrack(() => {
if (extendedFlowGraphAssetsCtx && _flowGraphAssetsCtx) {
const inputAssets = parseInputArgsAssets(job?.args ?? {})
const resourceMetadataCache = _flowGraphAssetsCtx.val.resourceMetadataCache
for (const asset of inputAssets) {
if (asset.kind === 'resource' && !(asset.path in resourceMetadataCache)) {
resourceMetadataCache[asset.path] = undefined
ResourceService.getResource({
workspace: workspace ?? $workspaceStore!,
path: asset.path
})
.then((r) => (resourceMetadataCache[asset.path] = r))
.catch((err) => {})
}
}
extendedFlowGraphAssetsCtx.val = clone(_flowGraphAssetsCtx?.val)
extendedFlowGraphAssetsCtx.val.additionalAssetsMap['Input'] = inputAssets
}
})
})
let jobResults: any[] = $state(
flowJobIds?.flowJobs?.map((x, id) => `iter #${id + 1} not loaded by frontend yet`) ?? []
)
let retry_selected = $state('')
let timeout: NodeJS.Timeout | undefined = undefined
let expandedSubflows: Record<string, FlowModule[]> = {}
$: flowJobIds?.moduleId && onFlowModuleId()
let expandedSubflows: Record<string, FlowModule[]> = $state({})
let selectedId: Writable<string | undefined> = writable(selectedNode)
@@ -328,7 +365,7 @@
}
}
let innerModules: FlowStatusModule[] = []
let innerModules: FlowStatusModule[] = $state([])
function updateStatus(status: FlowStatus) {
innerModules =
@@ -436,8 +473,6 @@
}
}
$: isForloopSelected && globalModuleStates && debounceLoadJobInProgress()
async function getNewJob(jobId: string, initialJob: Job | undefined) {
if (
jobId == initialJob?.id &&
@@ -446,11 +481,12 @@
) {
return initialJob
} else {
return await JobService.getJob({
let r = await JobService.getJob({
workspace: workspaceId ?? $workspaceStore ?? '',
id: jobId ?? '',
noLogs: true
})
return r
}
}
@@ -474,7 +510,7 @@
}
let errorCount = 0
let notAnonynmous = false
let notAnonynmous = $state(false)
let started = false
async function loadJobInProgress() {
if (!started) {
@@ -546,20 +582,15 @@
}
}
$: jobId && updateJobId()
$: isListJob = flowJobIds != undefined && Array.isArray(flowJobIds?.flowJobs)
function getTopModuleStates() {
return get(globalModuleStates?.[globalModuleStates?.length - 1])
}
let forloop_selected = getTopModuleStates()?.[flowJobIds?.moduleId ?? '']?.selectedForloop
let forloop_selected = $state(getTopModuleStates()?.[flowJobIds?.moduleId ?? '']?.selectedForloop)
let sub: Unsubscriber | undefined = undefined
let timeoutForloopSelectedSub: NodeJS.Timeout | undefined = undefined
let timeoutForloopSelected: NodeJS.Timeout | undefined = undefined
$: flowJobIds?.moduleId && onModuleIdChange()
function onModuleIdChange() {
clearTimeout(timeoutForloopSelectedSub)
@@ -583,8 +614,6 @@
sub?.()
})
$: selected = isListJob ? 'sequence' : 'graph'
function isSuccess(arg: any): boolean | undefined {
if (arg == undefined) {
return undefined
@@ -815,7 +844,7 @@
}
}
let flowTimeline: FlowTimeline
let flowTimeline: FlowTimeline | undefined = $state()
function loadPreviousIters(lenToAdd: number) {
let r = $localDurationStatuses[flowJobIds?.moduleId ?? '']
@@ -831,10 +860,10 @@
// updateSlicedListJobIds()
}
let stepDetail: FlowModule | string | undefined = undefined
let stepDetail: FlowModule | string | undefined = $state(undefined)
let storedListJobs: Record<number, Job> = {}
let wrapperHeight: number = 0
let storedListJobs: Record<number, Job> = $state({})
let wrapperHeight: number = $state(0)
function removeFailureNode(id: string, parent_module: any) {
if (id?.startsWith('failure-') && parent_module) {
@@ -883,7 +912,22 @@
return rec(ids, undefined)
}
let subflowsSize = 500
let subflowsSize = $state(500)
$effect(() => {
flowJobIds?.moduleId && untrack(() => onFlowModuleId())
})
$effect(() => {
isForloopSelected && globalModuleStates && untrack(() => debounceLoadJobInProgress())
})
$effect(() => {
jobId && untrack(() => updateJobId())
})
let isListJob = $derived(flowJobIds != undefined && Array.isArray(flowJobIds?.flowJobs))
$effect(() => {
flowJobIds?.moduleId && untrack(() => onModuleIdChange())
})
let selected = $derived(isListJob ? 'sequence' : 'graph')
</script>
{#if notAnonynmous}
@@ -906,7 +950,7 @@
<p class="text-tertiary italic text-xs">
For performance reasons, only the last 20 items are shown by default <button
class="text-primary underline ml-4"
on:click={() => {
onclick={() => {
loadPreviousIters(lenToAdd)
}}
>Load {lenToAdd} prior
@@ -915,7 +959,7 @@
{sliceFrom}
<button
class="text-primary underline ml-4"
on:click={() => {
onclick={() => {
loadPreviousIters(allToAdd)
}}
>Load {allToAdd} prior
@@ -1008,7 +1052,7 @@
(innerModule?.type != 'forloopflow' && innerModule?.type != 'whileloopflow')}
<!-- <LogId id={loopJobId} /> -->
<div class="border p-6" class:hidden={forloop_selected != loopJobId}>
<svelte:self
<FlowStatusViewerInner
{globalRefreshes}
parentRecursiveRefresh={recursiveRefresh}
{childFlow}
@@ -1027,7 +1071,7 @@
isForloopSelected={forloop_selected == loopJobId &&
(innerModule?.type == 'forloopflow' || innerModule?.type == 'whileloopflow')}
reducedPolling={reducedPolling ||
(flowJobIds?.flowJobs.length && flowJobIds?.flowJobs.length > 20)}
(!!flowJobIds?.flowJobs.length && flowJobIds?.flowJobs.length > 20)}
{workspaceId}
jobId={loopJobId}
on:jobsLoaded={(e) => {
@@ -1097,7 +1141,7 @@
<!-- <LogId id={loopJobId} /> -->
<div class="border p-6" class:hidden={retry_selected != failedRetry}>
<svelte:self
<FlowStatusViewerInner
{globalRefreshes}
parentRecursiveRefresh={recursiveRefresh}
{childFlow}
@@ -1116,7 +1160,7 @@
{/if}
{#if ['InProgress', 'Success', 'Failure'].includes(mod.type)}
{#if job.raw_flow?.modules[i]?.value.type == 'flow'}
<svelte:self
<FlowStatusViewerInner
{globalRefreshes}
parentRecursiveRefresh={recursiveRefresh}
globalModuleStates={[]}
@@ -1134,7 +1178,7 @@
]}
render={selected == 'sequence' && render}
{workspaceId}
jobId={mod.job}
jobId={mod.job ?? ''}
{reducedPolling}
isSubflow
childFlow
@@ -1146,7 +1190,7 @@
{:else if mod.flow_jobs?.length == 0 && mod.job == '00000000-0000-0000-0000-000000000000'}
<div class="text-secondary">no subflow (empty loop?)</div>
{:else}
<svelte:self
<FlowStatusViewerInner
{globalRefreshes}
parentRecursiveRefresh={recursiveRefresh}
{childFlow}
@@ -1157,14 +1201,14 @@
{prefix}
{subflowParentsGlobalModuleStates}
{subflowParentsDurationStatuses}
jobId={mod.job}
jobId={mod.job ?? ''}
{reducedPolling}
innerModule={mod.flow_jobs ? job.raw_flow?.modules[i]?.value : undefined}
flowJobIds={mod.flow_jobs
? {
moduleId: mod.id,
moduleId: mod.id ?? '',
flowJobs: mod.flow_jobs,
flowJobsSuccess: mod.flow_jobs_success,
flowJobsSuccess: mod.flow_jobs_success ?? [],
length: mod.iterator?.itered?.length ?? mod.flow_jobs.length,
branchall: job?.raw_flow?.modules?.[i]?.value?.type == 'branchall'
}
@@ -1212,7 +1256,6 @@
</div>
<FlowGraphV2
{inputAssets}
{selectedId}
triggerNode={true}
download={!hideDownloadInGraph}
frontend/src/lib/components/ScriptBuilder.svelte
+4 -3
View File
@@ -525,7 +525,7 @@
has_preprocessor: script.has_preprocessor,
deployment_message: deploymentMsg || undefined,
on_behalf_of_email: script.on_behalf_of_email,
fallback_access_types: script.fallback_access_types
assets: script.assets
}
})
@@ -666,7 +666,8 @@
visible_to_runner_only: script.visible_to_runner_only,
no_main_func: script.no_main_func,
has_preprocessor: script.has_preprocessor,
on_behalf_of_email: script.on_behalf_of_email
on_behalf_of_email: script.on_behalf_of_email,
assets: script.assets
}
})
}
@@ -1776,12 +1777,12 @@
kind={script.kind}
{template}
tag={script.tag}
bind:fallbackAccessTypes={script.fallback_access_types}
lastSavedCode={savedScript?.draft?.content}
lastDeployedCode={savedScript?.draft_only ? undefined : savedScript?.content}
bind:args
bind:hasPreprocessor
bind:captureTable
bind:assets={script.assets}
/>
</div>
{:else}
frontend/src/lib/components/ScriptEditor.svelte
+16 -34
View File
@@ -2,14 +2,7 @@
import { BROWSER } from 'esm-env'
import type { Schema, SupportedLanguage } from '$lib/common'
import {
AssetService,
type CompletedJob,
type Job,
JobService,
type Preview,
type ScriptLang
} from '$lib/gen'
import { type CompletedJob, type Job, JobService, type Preview, type ScriptLang } from '$lib/gen'
import { copilotInfo, enterpriseLicense, userStore, workspaceStore } from '$lib/stores'
import { copyToClipboard, emptySchema, sendUserToast } from '$lib/utils'
import Editor from './Editor.svelte'
@@ -54,8 +47,7 @@
import { aiChatManager, AIMode } from './copilot/chat/AIChatManager.svelte'
import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte'
import AssetsDropdownButton from './assets/AssetsDropdownButton.svelte'
import { usePromise } from '$lib/svelte5Utils.svelte'
import { assetEq, type AssetWithAccessType } from './assets/lib'
import { assetEq, type AssetWithAltAccessType } from './assets/lib'
interface Props {
// Exported
@@ -86,8 +78,8 @@
lastSavedCode?: string | undefined
lastDeployedCode?: string | undefined
disableAi?: boolean
assets?: AssetWithAltAccessType[]
editor_bar_right?: import('svelte').Snippet
fallbackAccessTypes?: AssetWithAccessType[]
}
let {
@@ -117,8 +109,8 @@
lastSavedCode = undefined,
lastDeployedCode = undefined,
disableAi = false,
editor_bar_right,
fallbackAccessTypes = $bindable()
assets = $bindable(),
editor_bar_right
}: Props = $props()
$effect.pre(() => {
@@ -147,28 +139,18 @@
dispatch('change', { code, schema })
})
let parsedAssets = usePromise(() => inferAssets(lang, code), { clearValueOnRefresh: false })
$effect(() => {
untrack(() => parsedAssets.refresh()), [lang, code]
})
// Load initial fallbackAccessTypes
if (edit && path) {
AssetService.listAssetsByUsage({
workspace: $workspaceStore!,
requestBody: { usages: [{ path, kind: 'script' }] }
}).then((arr) => {
const v = arr[0]
setTimeout(() => {
for (const a of parsedAssets.value ?? []) {
const fallback = v.find((a2) => assetEq(a2, a))?.access_type
if (!a.access_type && fallback) {
fallbackAccessTypes = [...(fallbackAccessTypes ?? []), { ...a, access_type: fallback }]
}
;[lang, code]
untrack(() => {
inferAssets(lang, code).then((newAssets: AssetWithAltAccessType[]) => {
for (const asset of newAssets) {
const old = assets?.find((a) => assetEq(a, asset))
if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
}
}, 200)
assets = newAssets
})
})
}
})
let width = $state(1200)
@@ -541,8 +523,8 @@
<Pane bind:size={codePanelSize} minSize={10} class="!overflow-visible">
<div class="h-full !overflow-visible bg-gray-50 dark:bg-[#272D38] relative">
<div class="absolute top-2 right-4 z-10 flex flex-row gap-2">
{#if parsedAssets.value?.length}
<AssetsDropdownButton assets={parsedAssets.value} bind:fallbackAccessTypes />
{#if assets?.length}
<AssetsDropdownButton {assets} />
{/if}
{#if testPanelSize === 0}
<HideButton
frontend/src/lib/components/assets/AssetButtons.svelte
+56
View File
@@ -0,0 +1,56 @@
<script lang="ts">
import { AlertTriangle, Edit2 } from 'lucide-svelte'
import { Button } from '../common'
import { Tooltip } from '../meltComponents'
import ExploreAssetButton, { assetCanBeExplored } from '../ExploreAssetButton.svelte'
import S3FilePicker from '../S3FilePicker.svelte'
import DbManagerDrawer from '../DBManagerDrawer.svelte'
import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte'
import type { Asset } from '$lib/gen'
type Props = {
s3FilePicker?: S3FilePicker | undefined
dbManagerDrawer?: DbManagerDrawer | undefined
resourceEditorDrawer?: ResourceEditorDrawer | undefined
resourceDataCache: Record<string, string | undefined>
asset: Asset
onClick?: () => void
}
let {
s3FilePicker,
dbManagerDrawer,
resourceEditorDrawer,
resourceDataCache,
asset,
onClick
}: Props = $props()
</script>
<div class="flex gap-2 items-center">
{#if asset.kind === 'resource' && resourceDataCache[asset.path] !== undefined}
<Button
startIcon={{ icon: Edit2 }}
size="xs"
variant="border"
spacingSize="xs2"
iconOnly
on:click={() => (resourceEditorDrawer?.initEdit(asset.path), onClick?.())}
/>
{/if}
{#if asset.kind === 'resource' && resourceDataCache[asset.path] === undefined}
<Tooltip class="mr-2.5">
<AlertTriangle size={16} class="text-orange-600 dark:text-orange-500" />
<svelte:fragment slot="text">Could not find resource</svelte:fragment>
</Tooltip>
{/if}
{#if assetCanBeExplored(asset, { resource_type: resourceDataCache[asset.path] })}
<ExploreAssetButton
{asset}
{s3FilePicker}
{dbManagerDrawer}
onClick={() => onClick?.()}
noText
_resourceMetadata={{ resource_type: resourceDataCache[asset.path] }}
/>
{/if}
</div>
frontend/src/lib/components/assets/AssetsDropdownButton.svelte
+67 -74
View File
@@ -1,20 +1,26 @@
<script lang="ts">
import { clone, pluralize } from '$lib/utils'
import { deepEqual } from 'fast-equals'
import { AlertTriangle, Edit2, Pyramid } from 'lucide-svelte'
import { Pyramid } from 'lucide-svelte'
import { twMerge } from 'tailwind-merge'
import { Popover } from '../meltComponents'
import S3FilePicker from '../S3FilePicker.svelte'
import ExploreAssetButton, { assetCanBeExplored } from '../ExploreAssetButton.svelte'
import { assetEq, formatAssetKind, type Asset, type AssetWithAccessType } from './lib'
import {
assetsEq,
formatAssetAccessType,
formatAssetKind,
getAccessType,
type Asset,
type AssetWithAltAccessType
} from './lib'
import DbManagerDrawer from '../DBManagerDrawer.svelte'
import { tick, untrack } from 'svelte'
import { untrack } from 'svelte'
import { ResourceService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import Button from '../common/button/Button.svelte'
import Tooltip from '../meltComponents/Tooltip.svelte'
import Tooltip2 from '../Tooltip.svelte'
import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte'
import type { Placement } from '@floating-ui/core'
import AssetButtons from './AssetButtons.svelte'
import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte'
@@ -25,17 +31,15 @@
noBtnText = false,
popoverPlacement = 'bottom-end',
disableLiTooltip = false,
fallbackAccessTypes = $bindable(),
onHoverLi,
liSubtitle
}: {
assets: AssetWithAccessType[]
assets: AssetWithAltAccessType[]
enableChangeAnimation?: boolean
size?: 'xs' | '3xs'
noBtnText?: boolean
popoverPlacement?: Placement
disableLiTooltip?: boolean
fallbackAccessTypes?: AssetWithAccessType[]
onHoverLi?: (asset: Asset, eventType: 'enter' | 'leave') => void
liSubtitle?: (asset: Asset) => string
} = $props()
@@ -60,7 +64,7 @@
$effect(() => {
assets
untrack(() => {
if (deepEqual(assets, prevAssets)) return
if (assetsEq(assets, prevAssets)) return
prevAssets = clone(assets)
// Replay animation
@@ -113,16 +117,52 @@
<svelte:fragment slot="content">
<ul class="divide-y rounded-md">
{#each assets as asset}
{@const fallbackAccessType = fallbackAccessTypes?.find((a) =>
assetEq(a, asset)
)?.access_type}
{@const hasWarning = !asset.access_type && !fallbackAccessType}
<li
class="text-sm px-4 h-12 flex gap-4 items-center justify-between hover:bg-surface-hover"
class="text-sm px-3 h-12 flex gap-3 items-center hover:bg-surface-hover/25"
onmouseenter={() => onHoverLi?.(asset, 'enter')}
onmouseleave={() => onHoverLi?.(asset, 'leave')}
>
<div class="flex flex-col">
<Popover
contentClasses="py-2 px-4 flex flex-col gap-2"
disablePopup={!!asset.access_type}
>
<svelte:fragment slot="trigger">
<div
class={twMerge(
'text-xs font-normal border text-tertiary w-10 p-1 text-center rounded-md',
!asset.access_type && !asset.alt_access_type
? 'text-orange-500 !border-orange-500'
: '',
!asset.access_type ? 'hover:bg-surface active:opacity-80' : ''
)}
>
{formatAssetAccessType(getAccessType(asset))}
</div>
</svelte:fragment>
<svelte:fragment slot="content">
{#if !asset.access_type}
<span class="text-sm text-tertiary leading-4">
Could not infer automatically <br />
<span class="text-xs">Please select manually </span>
</span>
<div class="flex items-center gap-2">
<ToggleButtonGroup bind:selected={asset.alt_access_type} class="max-w-fit">
{#snippet children({ item })}
<ToggleButton value="r" label="Read" {item} />
<ToggleButton value="w" label="Write" {item} />
<ToggleButton value="rw" label="Read/Write" {item} />
{/snippet}
</ToggleButtonGroup>
<Tooltip2>
This is used to determine if the asset should be displayed as an input or an
output node in the flow editor
</Tooltip2>
</div>
{/if}
</svelte:fragment>
</Popover>
<div class="flex flex-col flex-1">
<Tooltip class="select-none max-w-48 truncate" disablePopup={disableLiTooltip}>
{asset.path}
<svelte:fragment slot="text">
@@ -133,68 +173,21 @@
{liSubtitle?.(asset) ??
formatAssetKind({
...asset,
metadata: { resource_type: resourceDataCache[asset.path] }
...(asset.kind === 'resource'
? { metadata: { resource_type: resourceDataCache[asset.path] } }
: {})
})}
</span>
</div>
<div class="flex gap-2 items-center">
{#if asset.kind === 'resource' && resourceDataCache[asset.path] !== undefined}
<Button
startIcon={{ icon: Edit2 }}
size="xs"
variant="border"
spacingSize="xs2"
iconOnly
on:click={() => (resourceEditorDrawer?.initEdit(asset.path), (isOpen = false))}
/>
{/if}
{#if asset.kind === 'resource' && resourceDataCache[asset.path] === undefined}
<Tooltip class="mr-2.5">
<AlertTriangle size={16} class="text-orange-600 dark:text-orange-500" />
<svelte:fragment slot="text">Could not find resource</svelte:fragment>
</Tooltip>
{/if}
{#if assetCanBeExplored(asset, { resource_type: resourceDataCache[asset.path] })}
<ExploreAssetButton
{asset}
{s3FilePicker}
{dbManagerDrawer}
onClick={() => (isOpen = false)}
noText
_resourceMetadata={{ resource_type: resourceDataCache[asset.path] }}
/>
{/if}
<ToggleButtonGroup
disabled={!!asset.access_type}
tabListClass={hasWarning ? 'bg-red-200 dark:bg-red-300' : ''}
bind:selected={
() => asset.access_type ?? fallbackAccessType,
async (access_type) => {
fallbackAccessTypes ??= []
await tick()
let val = fallbackAccessTypes?.filter((a) => !assetEq(a, asset))
val.push({ ...asset, access_type })
fallbackAccessTypes = val
}
}
>
{#snippet children({ item })}
{#each ['r', 'w', 'rw'] as v}
<ToggleButton
class={hasWarning
? 'bg-transparent hover:bg-red-100 dark:text-primary-inverse'
: ''}
value={v}
label={v}
{item}
tooltip={'Could not infer access type from code, please select manually'}
/>
{/each}
{/snippet}
</ToggleButtonGroup>
</div>
<AssetButtons
onClick={() => (isOpen = false)}
{asset}
{resourceDataCache}
{dbManagerDrawer}
{resourceEditorDrawer}
{s3FilePicker}
/>
</li>
{/each}
</ul>
frontend/src/lib/components/assets/AssetsUsageDrawer.svelte
+9 -12
View File
@@ -1,12 +1,9 @@
<script lang="ts">
import type { AssetUsageAccessType, AssetUsageKind } from '$lib/gen'
import { twMerge } from 'tailwind-merge'
import { Drawer, DrawerContent } from '../common'
import RowIcon from '../common/table/RowIcon.svelte'
import {
assetDisplaysAsInputInFlowGraph,
assetDisplaysAsOutputInFlowGraph
} from '../graph/renderers/nodes/AssetNode.svelte'
import { getAssetUsagePageUri } from './lib'
import { formatAssetAccessType, getAssetUsagePageUri } from './lib'
let usagesDrawerData:
| {
@@ -42,13 +39,13 @@
<span class="font-semibold">{u.path}</span>
<span class="text-xs text-tertiary">{u.kind}</span>
</div>
<div class="flex gap-2">
{#if assetDisplaysAsInputInFlowGraph(u)}
<div class="text-xs border text-tertiary max-w-fit p-1 rounded-md">Read</div>
{/if}
{#if assetDisplaysAsOutputInFlowGraph(u)}
<div class="text-xs border text-tertiary max-w-fit p-1 rounded-md">Write</div>
{/if}
<div
class={twMerge(
'text-xs font-normal border text-tertiary w-10 p-1 text-center rounded-md',
!u.access_type ? 'hover:bg-surface active:opacity-80' : ''
)}
>
{formatAssetAccessType(u.access_type)}
</div>
</a>
</li>
frontend/src/lib/components/assets/JobAssetsViewer.svelte
+95
View File
@@ -0,0 +1,95 @@
<script lang="ts">
import { ResourceService, type Job } from '$lib/gen'
import { inferAssets } from '$lib/infer'
import { workspaceStore } from '$lib/stores'
import { usePromise } from '$lib/svelte5Utils.svelte'
import { pruneNullishArrayWithSet, uniqueBy } from '$lib/utils'
import DbManagerDrawer from '../DBManagerDrawer.svelte'
import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte'
import S3FilePicker from '../S3FilePicker.svelte'
import AssetButtons from './AssetButtons.svelte'
import {
formatAssetKind,
getFlowModuleAssets,
parseInputArgsAssets,
type AssetWithAccessType
} from './lib'
type Props = {
job: Job
}
let { job }: Props = $props()
async function extractAssets(job: Job): Promise<AssetWithAccessType[]> {
if (job.job_kind === 'flow') {
const additionalAssetsMap = {
// TODO : Transitive assets
}
return uniqueBy(
pruneNullishArrayWithSet([
...(job.raw_flow?.modules.flatMap((m) => getFlowModuleAssets(m, additionalAssetsMap)) ??
[]),
...parseInputArgsAssets(job.args ?? {})
]),
(x) => x.kind + x.path
)
}
if (job.job_kind === 'script') {
return [
...(await inferAssets(job.language!, job.raw_code ?? '')),
...parseInputArgsAssets(job.args ?? {})
]
}
return []
}
let assets = usePromise(() => extractAssets(job), { loadInit: false })
$effect(() => {
job.id
$workspaceStore
assets.refresh()
})
let resourceDataCache: Record<string, string | undefined> = $state({})
$effect(() => {
for (const asset of assets.value ?? []) {
if (asset.kind !== 'resource' || asset.path in resourceDataCache) continue
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! })
.then((resource) => (resourceDataCache[asset.path] = resource.resource_type))
.catch((err) => (resourceDataCache[asset.path] = undefined))
}
})
let s3FilePicker: S3FilePicker | undefined = $state()
let dbManagerDrawer: DbManagerDrawer | undefined = $state()
let resourceEditorDrawer: ResourceEditorDrawer | undefined = $state()
</script>
<ul class="flex flex-col divide-y mt-1">
{#each assets.value ?? [] as asset}
<li class="flex justify-between py-3 leading-4 text-sm pl-4">
<div class="flex flex-col flex-1 truncate">
{asset.path}
<span class="text-2xs text-tertiary">
{formatAssetKind({
...asset,
...(asset.kind === 'resource'
? { metadata: { resource_type: resourceDataCache[asset.path] } }
: {})
})}
</span>
</div>
<AssetButtons
{asset}
{resourceDataCache}
{dbManagerDrawer}
{resourceEditorDrawer}
{s3FilePicker}
/>
</li>
{/each}
</ul>
<S3FilePicker bind:this={s3FilePicker} readOnlyMode />
<DbManagerDrawer bind:this={dbManagerDrawer} />
<ResourceEditorDrawer bind:this={resourceEditorDrawer} />
frontend/src/lib/components/assets/README_DEV.md
View File
frontend/src/lib/components/assets/lib.ts
+55 -8
View File
@@ -2,13 +2,17 @@ import type {
AssetKind as _AssetKind,
Asset as _Asset,
ListAssetsResponse,
AssetUsageAccessType
AssetUsageAccessType,
FlowModule,
ScriptArgs
} from '$lib/gen'
import { capitalize } from '$lib/utils'
export type Asset = _Asset
export type AssetKind = _AssetKind
export type AssetWithAccessType = Asset & { access_type?: AssetUsageAccessType }
export type AssetWithAltAccessType = AssetWithAccessType & {
alt_access_type?: AssetUsageAccessType
}
export function formatAsset(asset: Asset): string {
switch (asset.kind) {
@@ -16,8 +20,6 @@ export function formatAsset(asset: Asset): string {
return `res://${asset.path}`
case 's3object':
return `s3://${asset.path}`
case 'variable':
return `var://${asset.path}`
}
}
@@ -34,6 +36,11 @@ export function assetEq(a: Asset | undefined, b: Asset | undefined): boolean {
return a.kind === b.kind && a.path === b.path
}
export function assetsEq(a: Asset[], b: Asset[]): boolean {
if (a.length !== b.length) return false
return a.every((asset, index) => assetEq(asset, b[index]))
}
export function parseAssetFromString(s: string): Asset | undefined {
if (s.startsWith('res://')) {
return { kind: 'resource', path: s.slice(6) }
@@ -41,8 +48,6 @@ export function parseAssetFromString(s: string): Asset | undefined {
return { kind: 'resource', path: s.slice(5) }
} else if (s.startsWith('s3://')) {
return { kind: 's3object', path: s.slice(5) }
} else if (s.startsWith('var://')) {
return { kind: 'variable', path: s.slice(6) }
}
return undefined
}
@@ -63,7 +68,49 @@ export function formatAssetKind(asset: {
}
case 's3object':
return 'S3 Object'
case 'variable':
return 'Variable'
}
}
export function formatAssetAccessType(accessType: AssetUsageAccessType | undefined) {
switch (accessType) {
case 'r':
return 'Read'
case 'w':
return 'Write'
case 'rw':
return 'R/W'
}
return '?'
}
export function getAccessType(asset: AssetWithAltAccessType): AssetUsageAccessType | undefined {
if (asset.alt_access_type) return asset.alt_access_type
if (asset.access_type) return asset.access_type
}
export function getFlowModuleAssets(
flowModuleValue: FlowModule,
additionalAssetsMap?: Record<string, AssetWithAccessType[]>
): AssetWithAccessType[] | undefined {
if (flowModuleValue.value.type === 'rawscript') return flowModuleValue.value.assets
if (flowModuleValue.value.type === 'script' || flowModuleValue.value.type === 'flow') {
const additionalAssets = additionalAssetsMap?.[flowModuleValue.id]
if (additionalAssets) return additionalAssets
}
return undefined
}
export function parseInputArgsAssets(args: ScriptArgs): AssetWithAccessType[] {
const arr: AssetWithAccessType[] = []
for (const v of Object.values(args)) {
if (typeof v === 'string') {
const asset = parseAssetFromString(v)
if (asset) arr.push(asset)
} else if (v && typeof v === 'object' && typeof v['s3'] === 'string') {
const s3 = v['s3']
const storage = typeof v['storage'] == 'string' ? v['storage'] : undefined
arr.push({ kind: 's3object', path: `${storage ?? ''}/${s3}` })
}
}
return arr
}
frontend/src/lib/components/flows/FlowAssetsHandler.svelte
+162
View File
@@ -0,0 +1,162 @@
<script lang="ts" module>
export function initFlowGraphAssetsCtx({
getModules
}: {
getModules: () => FlowModule[]
}): FlowGraphAssetContext {
let s = $state({
val: {
selectedAsset: undefined,
dbManagerDrawer: undefined,
s3FilePicker: undefined,
resourceEditorDrawer: undefined,
resourceMetadataCache: {},
additionalAssetsMap: {},
computeAssetsCount: (asset) => {
return getAllModules(getModules())
.flatMap((m) => getFlowModuleAssets(m, s.val.additionalAssetsMap) ?? [])
.filter((a) => assetEq(asset, a)).length
}
}
} satisfies FlowGraphAssetContext)
return s
}
</script>
<script lang="ts">
import { inferAssets } from '$lib/infer'
import {
assetEq,
getFlowModuleAssets,
type AssetWithAccessType,
type AssetWithAltAccessType
} from '../assets/lib'
import OnChange from '../common/OnChange.svelte'
import { getAllModules } from './flowExplorer'
import { getContext, untrack } from 'svelte'
import type { FlowGraphAssetContext } from './types'
import {
AssetService,
ResourceService,
type AssetUsageKind,
type FlowModule,
type RawScript
} from '$lib/gen'
import { deepEqual } from 'fast-equals'
import { workspaceStore } from '$lib/stores'
import S3FilePicker from '../S3FilePicker.svelte'
import DbManagerDrawer from '../DBManagerDrawer.svelte'
import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte'
let {
modules,
enableParser = false,
enableDbExplore = false,
enablePathScriptAndFlowAssets = false
}: {
modules: FlowModule[]
enableParser?: boolean
enableDbExplore?: boolean
enablePathScriptAndFlowAssets?: boolean
} = $props()
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
let allModules = $derived(getAllModules(modules))
// Fetch resource metadata for the ExploreAssetButton
const resMetadataCache = $derived(flowGraphAssetsCtx?.val.resourceMetadataCache)
$effect(() => {
if (!resMetadataCache || !enableDbExplore) return
const assets: AssetWithAccessType[] =
allModules.flatMap(
(m) => getFlowModuleAssets(m, flowGraphAssetsCtx?.val.additionalAssetsMap) ?? []
) ?? []
for (const asset of assets) {
if (asset.kind !== 'resource' || asset.path in resMetadataCache) continue
resMetadataCache[asset.path] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! })
.then((r) => (resMetadataCache[asset.path] = { resource_type: r.resource_type }))
.catch((err) => {
console.error("Couldn't fetch resource", asset.path, err)
})
}
})
// Fetch transitive assets (path scripts and flows)
$effect(() => {
if (!$workspaceStore || !flowGraphAssetsCtx || !enablePathScriptAndFlowAssets) return
let usages: { path: string; kind: AssetUsageKind }[] = []
let modIds: string[] = []
for (const mod of allModules) {
if (mod.id in flowGraphAssetsCtx.val.additionalAssetsMap) continue
flowGraphAssetsCtx.val.additionalAssetsMap[mod.id] = [] // avoid fetching multiple times because of async
if (mod.value.type === 'flow' || mod.value.type === 'script') {
usages.push({ path: mod.value.path, kind: mod.value.type })
modIds.push(mod.id)
}
}
if (usages.length) {
AssetService.listAssetsByUsage({
workspace: $workspaceStore,
requestBody: { usages }
}).then((result) => {
result.forEach((assets, idx) => {
flowGraphAssetsCtx.val.additionalAssetsMap[modIds[idx]] = assets
})
})
}
})
// Prune all additionalAssetsMap entries from deleted modules
$effect(() => {
if (!flowGraphAssetsCtx) return
const modulesSet = new Set(allModules.map((m) => m.id))
for (const key of Object.keys(flowGraphAssetsCtx.val.additionalAssetsMap)) {
if (!modulesSet.has(key)) {
delete flowGraphAssetsCtx.val.additionalAssetsMap[key]
}
}
})
async function parseAndUpdateRawScriptModule(v: RawScript) {
try {
let parsedAssets: AssetWithAltAccessType[] = await inferAssets(v.language, v.content)
for (const asset of parsedAssets) {
const old = v.assets?.find((a) => assetEq(a, asset))
if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type
}
if (!deepEqual(v.assets, parsedAssets)) v.assets = parsedAssets
} catch (e) {}
}
// Check for raw script modules whose assets were not parsed. Useful for flows created
// before the assets feature was introduced.
$effect(() => {
if (!enableParser) return
untrack(() => {
setTimeout(() => {
for (const mod of allModules) {
if (mod.value.type === 'rawscript' && mod.value.assets === undefined) {
console.log('RawScript module', mod.id, 'without assets field, parsing')
parseAndUpdateRawScriptModule(mod.value)
}
}
}, 500) // ensure modules are loaded
})
})
</script>
{#if enableParser}
{#each allModules as mod (mod.id)}
{#if mod.value.type === 'rawscript'}
{@const v = mod.value}
<OnChange key={v.content} onChange={() => parseAndUpdateRawScriptModule(v)} />
{/if}
{/each}
{/if}
{#if flowGraphAssetsCtx}
<S3FilePicker bind:this={flowGraphAssetsCtx.val.s3FilePicker} readOnlyMode />
<DbManagerDrawer bind:this={flowGraphAssetsCtx.val.dbManagerDrawer} />
<ResourceEditorDrawer bind:this={flowGraphAssetsCtx.val.resourceEditorDrawer} />
{/if}
frontend/src/lib/components/flows/content/FlowModuleComponent.svelte
+5 -20
View File
@@ -51,10 +51,9 @@
import { workspaceStore } from '$lib/stores'
import { checkIfParentLoop } from '../utils'
import ModulePreviewResultViewer from '$lib/components/ModulePreviewResultViewer.svelte'
import { refreshStateStore, usePromise } from '$lib/svelte5Utils.svelte'
import { refreshStateStore } from '$lib/svelte5Utils.svelte'
import { getStepHistoryLoaderContext } from '$lib/components/stepHistoryLoader.svelte'
import AssetsDropdownButton from '$lib/components/assets/AssetsDropdownButton.svelte'
import { inferAssets } from '$lib/infer'
const {
selectedId,
@@ -123,6 +122,8 @@
let testIsLoading = $state(false)
let scriptProgress = $state(undefined)
let assets = $derived((flowModule.value.type === 'rawscript' && flowModule.value.assets) || [])
function onModulesChange(savedModule: FlowModule | undefined, flowModule: FlowModule) {
// console.log('onModulesChange', savedModule, flowModule)
return savedModule?.value?.type === 'rawscript' &&
@@ -299,19 +300,6 @@
}
})
let assets = usePromise(
async () =>
flowModule.value.type === 'rawscript'
? await inferAssets(flowModule.value.language, flowModule.value.content)
: undefined,
{ clearValueOnRefresh: false, loadInit: false }
)
$effect(() => {
if (flowModule.value.type !== 'rawscript') return
;[flowModule.value.content, flowModule.value.language]
untrack(() => assets.refresh())
})
let rawScriptLang = $derived(
flowModule.value.type == 'rawscript' ? flowModule.value.language : undefined
)
@@ -422,11 +410,8 @@
{#if !noEditor}
{#key flowModule.id}
<div class="absolute top-2 right-4 z-10 flex flex-row gap-2">
{#if assets.value?.length}
<AssetsDropdownButton
assets={assets.value}
bind:fallbackAccessTypes={flowModule.value.asset_fallback_access_types}
/>
{#if assets?.length}
<AssetsDropdownButton {assets} />
{/if}
</div>
<Editor
frontend/src/lib/components/flows/types.ts
+2 -1
View File
@@ -89,10 +89,11 @@ export type FlowEditorContext = {
export type FlowGraphAssetContext = StateStore<{
selectedAsset: Asset | undefined
assetsMap: Record<string, AssetWithAccessType[]> // Maps module ids to their assets
s3FilePicker: S3FilePicker | undefined
dbManagerDrawer: DbManagerDrawer | undefined
resourceEditorDrawer: ResourceEditorDrawer | undefined
// Maps resource paths to their metadata. undefined is for error
resourceMetadataCache: Record<string, { resource_type?: string } | undefined>
additionalAssetsMap: Record<string, AssetWithAccessType[]>
computeAssetsCount: (asset: Asset) => number
}>
frontend/src/lib/components/graph/FlowGraphV2.svelte
+12 -116
View File
@@ -1,12 +1,5 @@
<script lang="ts">
import {
AssetService,
FlowService,
ResourceService,
type AssetUsageKind,
type FlowModule,
type Job
} from '../../gen'
import { FlowService, type FlowModule, type Job } from '../../gen'
import { NODE, type GraphModuleState } from '.'
import { getContext, onDestroy, setContext, tick, untrack } from 'svelte'
@@ -58,15 +51,8 @@
import { deepEqual } from 'fast-equals'
import ViewportResizer from './ViewportResizer.svelte'
import AssetNode, { computeAssetNodes } from './renderers/nodes/AssetNode.svelte'
import type { FlowGraphAssetContext } from '../flows/types'
import { getAllModules } from '../flows/flowExplorer'
import { inferAssets } from '$lib/infer'
import OnChange from '../common/OnChange.svelte'
import S3FilePicker from '../S3FilePicker.svelte'
import DbManagerDrawer from '../DBManagerDrawer.svelte'
import ResourceEditorDrawer from '../ResourceEditorDrawer.svelte'
import { assetEq, type AssetWithAccessType } from '../assets/lib'
import AssetsOverflowedNode from './renderers/nodes/AssetsOverflowedNode.svelte'
import type { FlowGraphAssetContext } from '../flows/types'
let useDataflow: Writable<boolean | undefined> = writable<boolean | undefined>(false)
@@ -103,7 +89,6 @@
editMode?: boolean
allowSimplifiedPoll?: boolean
expandedSubflows?: Record<string, FlowModule[]>
inputAssets?: AssetWithAccessType[]
isOwner?: boolean
isRunning?: boolean
individualStepTests?: boolean
@@ -174,7 +159,6 @@
editMode = false,
allowSimplifiedPoll = true,
expandedSubflows = $bindable({}),
inputAssets,
onTestUpTo = undefined,
onEditInput = undefined,
isOwner = false,
@@ -204,67 +188,6 @@
})
}
const flowGraphAssetsCtx: FlowGraphAssetContext = $state({
val: {
assetsMap: inputAssets ? ({ Input: inputAssets } as any) : {},
selectedAsset: undefined,
dbManagerDrawer: undefined,
s3FilePicker: undefined,
resourceEditorDrawer: undefined,
resourceMetadataCache: {}
}
})
setContext<FlowGraphAssetContext>('FlowGraphAssetContext', flowGraphAssetsCtx)
const assetsMap = $derived(flowGraphAssetsCtx.val.assetsMap)
$effect(() => {
if (inputAssets) flowGraphAssetsCtx.val.assetsMap.Input = inputAssets
})
// Fetch resource metadata for the ExploreAssetButton
const resMetadataCache = $derived(flowGraphAssetsCtx.val.resourceMetadataCache)
$effect(() => {
for (const asset of Object.values(assetsMap ?? []).flatMap((x) => x)) {
if (asset.kind !== 'resource' || asset.path in resMetadataCache) continue
resMetadataCache[asset.path] = undefined // avoid fetching multiple times because of async
ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! }).then(
(r) => (resMetadataCache[asset.path] = { resource_type: r.resource_type })
)
}
})
// Fetch transitive assets (path scripts and flows)
$effect(() => {
if (!$workspaceStore) return
let usages: { path: string; kind: AssetUsageKind }[] = []
let modIds: string[] = []
for (const mod of getAllModules(modules)) {
if (mod.id in assetsMap) continue
assetsMap[mod.id] = [] // avoid fetching multiple times because of async
if (mod.value.type === 'flow' || mod.value.type === 'script') {
usages.push({ path: mod.value.path, kind: mod.value.type })
modIds.push(mod.id)
}
}
if (usages.length) {
AssetService.listAssetsByUsage({
workspace: $workspaceStore,
requestBody: { usages }
}).then((result) => {
result.forEach((assets, idx) => {
assetsMap[modIds[idx]] = assets
})
})
}
})
// Prune assetsMap to only contain assets that are actually used
$effect(() => {
const allModules = new Set(getAllModules(modules).map((mod) => mod.id))
for (const modId in assetsMap) {
if (modId !== 'Input' && !allModules.has(modId)) delete assetsMap[modId]
}
})
function computeSimplifiableFlow(modules: FlowModule[], simplifiedFlow: boolean) {
const isSimplif = isSimplifiable(modules)
simplifiableFlow = isSimplif ? { simplifiedFlow } : undefined
@@ -283,8 +206,8 @@
)
}
let lastNodes: [NodeLayout[], Node[]] | undefined = undefined
function layoutNodes(nodes: NodeLayout[]): Node[] {
let lastNodes: [NodeLayout[], (Node & NodeLayout)[]] | undefined = undefined
function layoutNodes(nodes: NodeLayout[]): (Node & NodeLayout)[] {
let lastResult = lastNodes?.[1]
if (lastResult && nodes === lastNodes?.[0]) {
return lastResult
@@ -457,11 +380,8 @@
}
let newGraph = graph
newGraph.nodes.sort((a, b) => b.id.localeCompare(a.id))
;[nodes, edges] = computeAssetNodes(layoutNodes(newGraph.nodes), newGraph.edges, assetsMap, {
moving,
eventHandlers: eventHandler,
disableAi
})
console.log('compute')
;[nodes, edges] = computeAssetNodes(layoutNodes(newGraph.nodes), newGraph.edges)
await tick()
height = Math.max(...nodes.map((n) => n.position.y + NODE.height + 100), minHeight)
}
@@ -499,11 +419,14 @@
// })
let yamlEditorDrawer: Drawer | undefined = $state(undefined)
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
$effect(() => {
allowSimplifiedPoll && modules && untrack(() => onModulesChange(modules ?? []))
})
$effect(() => {
modules && untrack(() => onModulesChange2(modules))
readFieldsRecursively(modules)
untrack(() => onModulesChange2(modules))
})
let graph = $derived.by(() => {
moduleCounter
@@ -525,7 +448,8 @@
flowJob,
showJobStatus,
suspendStatus,
flowHasChanged
flowHasChanged,
additionalAssetsMap: flowGraphAssetsCtx?.val.additionalAssetsMap
},
failureModule,
preprocessorModule,
@@ -541,7 +465,6 @@
})
$effect(() => {
;[graph, allowSimplifiedPoll]
readFieldsRecursively(assetsMap)
untrack(() => updateStores())
})
@@ -670,33 +593,6 @@
{/if}
</div>
{#each getAllModules(modules) as mod (mod.id)}
{#if mod.value.type === 'rawscript'}
{@const v = mod.value}
<OnChange
key={[v.content, v.asset_fallback_access_types]}
runFirstEffect
onChange={() =>
inferAssets(v.language, v.content)
.then((assets) => {
for (const override of v.asset_fallback_access_types ?? []) {
assets = assets.map((asset) => {
if (assetEq(asset, override) && !asset.access_type)
return { ...asset, access_type: override.access_type }
return asset
})
}
if (assetsMap && !deepEqual(assetsMap[mod.id], assets)) assetsMap[mod.id] = assets
})
.catch((e) => {})}
/>
{/if}
{/each}
<S3FilePicker bind:this={flowGraphAssetsCtx.val.s3FilePicker} readOnlyMode />
<DbManagerDrawer bind:this={flowGraphAssetsCtx.val.dbManagerDrawer} />
<ResourceEditorDrawer bind:this={flowGraphAssetsCtx.val.resourceEditorDrawer} />
<style lang="postcss">
:global(.svelte-flow__handle) {
opacity: 0;
frontend/src/lib/components/graph/graphBuilder.svelte.ts
+24 -7
View File
@@ -1,11 +1,12 @@
import type { FlowModule, Job, RawScript, Script } from '$lib/gen'
import { type Edge } from '@xyflow/svelte'
import { getDependeeAndDependentComponents } from '../flows/flowExplorer'
import { getAllModules, getDependeeAndDependentComponents } from '../flows/flowExplorer'
import { dfsByModule } from '../flows/previousResults'
import { defaultIfEmptyString } from '$lib/utils'
import type { GraphModuleState } from './model'
import type { AssetWithAccessType } from '../assets/lib'
import { getFlowModuleAssets, type AssetWithAltAccessType } from '../assets/lib'
import type { Writable } from 'svelte/store'
import { assetDisplaysAsOutputInFlowGraph } from './renderers/nodes/AssetNode.svelte'
export type InsertKind =
| 'script'
@@ -115,6 +116,7 @@ export type InputN = {
flowJob: Job | undefined
showJobStatus: boolean
flowHasChanged: boolean
assets?: AssetWithAltAccessType[] | undefined
}
}
@@ -132,6 +134,7 @@ export type ModuleN = {
editMode: boolean
flowJob: Job | undefined
isOwner: boolean
assets: AssetWithAltAccessType[] | undefined
}
}
@@ -277,14 +280,14 @@ export type TriggerN = {
export type AssetN = {
type: 'asset'
data: {
asset: AssetWithAccessType
asset: AssetWithAltAccessType
}
}
export type AssetsOverflowedN = {
type: 'assetsOverflowed'
data: {
overflowedAssets: AssetWithAccessType[]
overflowedAssets: AssetWithAltAccessType[]
}
}
@@ -322,6 +325,7 @@ export function graphBuilder(
showJobStatus: boolean
suspendStatus: Writable<Record<string, { job: Job; nb: number }>>
flowHasChanged: boolean
additionalAssetsMap?: Record<string, AssetWithAltAccessType[]>
},
failureModule: FlowModule | undefined,
preprocessorModule: FlowModule | undefined,
@@ -374,7 +378,8 @@ export function graphBuilder(
insertable: extra.insertable,
editMode: extra.editMode,
isOwner: extra.isOwner,
flowJob: extra.flowJob
flowJob: extra.flowJob,
assets: getFlowModuleAssets(module, extra.additionalAssetsMap)
},
type: 'module'
})
@@ -382,6 +387,15 @@ export function graphBuilder(
return module.id
}
// TODO : Do better than this
const nodeIdsWithOutputAssets = new Set(
getAllModules(modules)
.filter((m) =>
getFlowModuleAssets(m, extra.additionalAssetsMap)?.some(assetDisplaysAsOutputInFlowGraph)
)
.map((m) => m.id)
)
const parents: { [key: string]: string[] } = {}
//
@@ -458,11 +472,13 @@ export function graphBuilder(
// If the index is -1, it means that the target module is not in the modules array, so we set it to the length of the array
index: index >= 0 ? index : (mods?.length ?? 0),
...extra,
insertable: extra.insertable && !options?.disableInsert && prefix == undefined
insertable: extra.insertable && !options?.disableInsert && prefix == undefined,
shouldOffsetInsertBtnDueToAssetNode: nodeIdsWithOutputAssets.has(sourceId)
}
})
}
const inputAssets = extra.additionalAssetsMap?.['Input']
const inputNode: NodeLayout = {
id: 'Input',
type: 'input2',
@@ -479,7 +495,8 @@ export function graphBuilder(
individualStepTests: extra.individualStepTests,
flowJob: extra.flowJob,
showJobStatus: extra.showJobStatus,
flowHasChanged: extra.flowHasChanged
flowHasChanged: extra.flowHasChanged,
...(inputAssets ? { assets: inputAssets } : {})
}
}
frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte
+3 -12
View File
@@ -7,11 +7,7 @@
import type { GraphEventHandlers } from '../../graphBuilder.svelte'
import { getStraightLinePath } from '../utils'
import { twMerge } from 'tailwind-merge'
import { type FlowGraphAssetContext } from '$lib/components/flows/types'
import {
assetDisplaysAsOutputInFlowGraph,
NODE_WITH_WRITE_ASSET_Y_OFFSET
} from '../nodes/AssetNode.svelte'
import { NODE_WITH_WRITE_ASSET_Y_OFFSET } from '../nodes/AssetNode.svelte'
import { workspaceStore } from '$lib/stores'
import FlowStatusWaitingForEvents from '$lib/components/FlowStatusWaitingForEvents.svelte'
import type { Job } from '$lib/gen'
@@ -21,8 +17,6 @@
useDataflow: Writable<boolean | undefined>
}>('FlowGraphContext')
const flowGraphAssetCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
let {
// id,
sourceX,
@@ -50,13 +44,10 @@
isOwner: boolean
flowJob: Job | undefined
suspendStatus?: Writable<Record<string, { job: Job; nb: number }>>
shouldOffsetInsertBtnDueToAssetNode?: boolean
}
} = $props()
const shouldOffsetInsertButtonDueToAssetNode = flowGraphAssetCtx?.val.assetsMap?.[
data.sourceId
]?.some(assetDisplaysAsOutputInFlowGraph)
let [edgePath] = $derived(
getBezierPath({
sourceX,
@@ -89,7 +80,7 @@
<EdgeLabel
x={sourceX}
y={sourceY + 28 + (shouldOffsetInsertButtonDueToAssetNode ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0)}
y={sourceY + 28 + (data.shouldOffsetInsertBtnDueToAssetNode ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0)}
class="base-edge"
style=""
>
frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte
+37 -34
View File
@@ -3,33 +3,31 @@
export const NODE_WITH_WRITE_ASSET_Y_OFFSET = 45
export const READ_ASSET_Y_OFFSET = -45
export const WRITE_ASSET_Y_OFFSET = 64
export const assetDisplaysAsInputInFlowGraph = (a: { access_type?: AssetUsageAccessType }) =>
!a.access_type || a.access_type === 'r' || a.access_type === 'rw'
export const assetDisplaysAsOutputInFlowGraph = (a: { access_type?: AssetUsageAccessType }) =>
a.access_type === 'w' || a.access_type === 'rw'
export const assetDisplaysAsInputInFlowGraph = (a: AssetWithAltAccessType) =>
!getAccessType(a) || getAccessType(a) === 'r' || getAccessType(a) === 'rw'
export const assetDisplaysAsOutputInFlowGraph = (a: AssetWithAltAccessType) =>
getAccessType(a) === 'w' || getAccessType(a) === 'rw'
let computeAssetNodesCache:
| [Node[], Record<string, AssetWithAccessType[]>, ReturnType<typeof computeAssetNodes>]
| [(Node & NodeLayout)[], ReturnType<typeof computeAssetNodes>]
| undefined
export function computeAssetNodes(
nodes: Node[],
edges: Edge[],
assetsMap: Record<string, AssetWithAccessType[]>,
extraData: any
): [Node[], Edge[]] {
if (nodes === computeAssetNodesCache?.[0] && deepEqual(assetsMap, computeAssetNodesCache?.[1]))
return computeAssetNodesCache[2]
nodes: (Node & NodeLayout)[],
edges: Edge[]
): [(Node & NodeLayout)[], Edge[]] {
if (nodes === computeAssetNodesCache?.[0]) return computeAssetNodesCache[1]
const MAX_ASSET_ROW_WIDTH = 300
const ASSETS_OVERFLOWED_NODE_WIDTH = 25
const allAssetNodes: Node[] = []
const allAssetNodes: (Node & NodeLayout)[] = []
const allAssetEdges: Edge[] = []
const yPosMap: Record<number, { r?: true; w?: true }> = {}
for (const node of nodes) {
const assets = assetsMap?.[node.id] ?? []
if (node.type !== 'module' && node.type !== 'input2') continue
const assets = node.data.assets ?? []
// Each asset can be displayed at the top and bottom
// i.e once (R or W) or twice (RW)
@@ -197,16 +195,21 @@
[...sortedNewNodes, ...allAssetNodes],
[...edges, ...allAssetEdges]
]
computeAssetNodesCache = [nodes, clone(assetsMap), ret]
computeAssetNodesCache = [nodes, ret]
return ret
}
</script>
<script lang="ts">
import NodeWrapper from './NodeWrapper.svelte'
import type { AssetN, AssetsOverflowedN } from '../../graphBuilder.svelte'
import type { AssetN, AssetsOverflowedN, NodeLayout } from '../../graphBuilder.svelte'
import { AlertTriangle } from 'lucide-svelte'
import { assetEq, formatAssetKind, type AssetWithAccessType } from '$lib/components/assets/lib'
import {
assetEq,
formatAssetKind,
getAccessType,
type AssetWithAltAccessType
} from '$lib/components/assets/lib'
import { twMerge } from 'tailwind-merge'
import type { FlowGraphAssetContext } from '$lib/components/flows/types'
import { getContext } from 'svelte'
@@ -215,28 +218,21 @@
import { clone, pluralize } from '$lib/utils'
import AssetGenericIcon from '$lib/components/icons/AssetGenericIcon.svelte'
import type { Edge, Node } from '@xyflow/svelte'
import { deepEqual } from 'fast-equals'
import { NODE } from '../../util'
import type { AssetUsageAccessType } from '$lib/gen'
import { userStore } from '$lib/stores'
interface Props {
data: AssetN['data']
}
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext>('FlowGraphAssetContext')
const usageCount = $derived(
Object.values(flowGraphAssetsCtx.val.assetsMap ?? {})
.flat()
.filter((asset) => assetEq(asset, data.asset)).length
)
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
let { data }: Props = $props()
const isSelected = $derived(assetEq(flowGraphAssetsCtx.val.selectedAsset, data.asset))
const isSelected = $derived(assetEq(flowGraphAssetsCtx?.val.selectedAsset, data.asset))
const cachedResourceMetadata = $derived(
flowGraphAssetsCtx.val.resourceMetadataCache[data.asset.path]
flowGraphAssetsCtx?.val.resourceMetadataCache[data.asset.path]
)
</script>
@@ -249,8 +245,10 @@
'bg-surface h-6 flex items-center gap-1.5 rounded-sm text-tertiary border overflow-clip',
isSelected ? 'bg-surface-secondary !border-surface-inverse' : 'border-transparent'
)}
onmouseenter={() => (flowGraphAssetsCtx.val.selectedAsset = data.asset)}
onmouseleave={() => (flowGraphAssetsCtx.val.selectedAsset = undefined)}
onmouseenter={() =>
flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = data.asset)}
onmouseleave={() =>
flowGraphAssetsCtx && (flowGraphAssetsCtx.val.selectedAsset = undefined)}
>
<AssetGenericIcon
assetKind={data.asset.kind}
@@ -272,14 +270,17 @@
asset={data.asset}
noText
buttonVariant="contained"
s3FilePicker={flowGraphAssetsCtx.val.s3FilePicker}
dbManagerDrawer={flowGraphAssetsCtx.val.dbManagerDrawer}
s3FilePicker={flowGraphAssetsCtx?.val.s3FilePicker}
dbManagerDrawer={flowGraphAssetsCtx?.val.dbManagerDrawer}
_resourceMetadata={cachedResourceMetadata}
/>
{/if}
</div>
<svelte:fragment slot="text">
Used in {pluralize(usageCount, 'step')}<br />
Used in {pluralize(
flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset) ?? -1,
'step'
)}<br />
<a
href={undefined}
class={twMerge(
@@ -290,7 +291,7 @@
)}
onclick={() => {
if (data.asset.kind === 'resource')
flowGraphAssetsCtx.val.resourceEditorDrawer?.initEdit(data.asset.path)
flowGraphAssetsCtx?.val.resourceEditorDrawer?.initEdit(data.asset.path)
}}
>
{data.asset.path}
@@ -298,6 +299,8 @@
<span class="dark:text-tertiary text-tertiary-inverse text-xs"
>{formatAssetKind({ ...data.asset, metadata: cachedResourceMetadata })}</span
>
<br />
fdsfs
</svelte:fragment>
</Tooltip>
{/snippet}
frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte
+2 -2
View File
@@ -14,12 +14,12 @@
data: AssetsOverflowedN['data']
}
let { data }: Props = $props()
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext>('FlowGraphAssetContext')
const flowGraphAssetsCtx = getContext<FlowGraphAssetContext | undefined>('FlowGraphAssetContext')
let isOpen = $state(false)
let includesSelected = $derived(
data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx.val.selectedAsset, asset))
data.overflowedAssets.some((asset) => assetEq(flowGraphAssetsCtx?.val.selectedAsset, asset))
)
let wasOpenedBecauseOfExternalSelected = false
frontend/src/lib/components/icons/AssetGenericIcon.svelte
-3
View File
@@ -3,7 +3,6 @@
import type { AssetKind } from '../assets/lib'
import AssetResIcon from './AssetResIcon.svelte'
import AssetS3Icon from './AssetS3Icon.svelte'
import AssetVarIcon from './AssetVarIcon.svelte'
interface Props {
size?: string
@@ -19,8 +18,6 @@
<AssetS3Icon {fill} width={size} height={size} class={className} />
{:else if assetKind == 'resource'}
<AssetResIcon {fill} width={size} height={size} class={className} />
{:else if assetKind == 'variable'}
<AssetVarIcon {fill} width={size} height={size} class={className} />
{:else}
<Pyramid {size} color={fill} class={'!fill-none ' + className} />
{/if}
frontend/src/lib/components/icons/AssetVarIcon.svelte
-38
View File
@@ -1,38 +0,0 @@
<script lang="ts">
interface Props {
height?: string
width?: string
fill?: string
class?: string
}
let { height = '24px', width = '24px', fill = 'black', class: className = '' }: Props = $props()
</script>
<svg
{width}
{height}
class={className}
viewBox="0 0 22 22"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<g clip-path="url(#clip0_1_2)">
<path
d="M11.0586 0.306641C11.3106 0.315989 11.5582 0.378277 11.7822 0.495117C12.0247 0.621655 12.2331 0.805033 12.3897 1.0293L12.3916 1.03223L17.9033 9H16.2022L11.6963 2.4873V20.0986L12.75 19.4961V21.1094L12.333 21.3477L12.332 21.3486C11.925 21.5804 11.4645 21.7021 10.9961 21.7021C10.5275 21.7021 10.0664 21.5806 9.65919 21.3486V21.3477L1.14844 16.4883L1.14258 16.4844C0.940872 16.3666 0.765826 16.2081 0.627936 16.0195C0.489969 15.8308 0.391887 15.6158 0.340827 15.3877C0.289772 15.1595 0.286682 14.9229 0.331061 14.6934C0.375447 14.4641 0.466395 14.2462 0.598639 14.0537L0.599616 14.0518L9.59962 1.03223L9.60157 1.0293C9.75823 0.804889 9.9673 0.62167 10.21 0.495117C10.4336 0.378564 10.6802 0.316161 10.9316 0.306641C10.9528 0.304686 10.9744 0.299805 10.9961 0.299805C11.0172 0.299841 11.038 0.304797 11.0586 0.306641ZM1.75196 14.8457C1.72867 14.8796 1.71294 14.9186 1.70508 14.959C1.69725 14.9995 1.69803 15.0418 1.70704 15.082C1.71605 15.1222 1.73351 15.1601 1.75782 15.1934C1.78213 15.2266 1.81309 15.2546 1.84864 15.2754H1.84766L10.2959 20.0986V2.4873L1.75196 14.8457Z"
{fill}
stroke="none"
/>
<path
d="M18 10.8496C18.1472 10.8496 18.2858 10.9138 18.3857 11.0225C18.4851 11.1306 18.539 11.2749 18.5391 11.4229V11.9785H20.5928C20.7397 11.9786 20.8777 12.042 20.9775 12.1504C21.077 12.2586 21.1318 12.4037 21.1318 12.5518C21.1317 12.6997 21.0769 12.844 20.9775 12.9521C20.8777 13.0606 20.7397 13.124 20.5928 13.124H18.5391V15.9268H19.2959C19.9244 15.9268 20.5246 16.1988 20.9648 16.6777C21.4047 17.1563 21.6504 17.8029 21.6504 18.4746C21.6503 19.1463 21.4046 19.793 20.9648 20.2715C20.5246 20.7503 19.9243 21.0215 19.2959 21.0215H18.5391V21.5771C18.539 21.7251 18.4851 21.8694 18.3857 21.9775C18.2858 22.0862 18.1472 22.1504 18 22.1504C17.8528 22.1504 17.7142 22.0862 17.6143 21.9775C17.5149 21.8694 17.461 21.7251 17.4609 21.5771V21.0215H14.8887C14.7417 21.0214 14.6037 20.958 14.5039 20.8496C14.4044 20.7414 14.3496 20.5963 14.3496 20.4482C14.3497 20.3003 14.4045 20.156 14.5039 20.0479C14.6037 19.9394 14.7417 19.876 14.8887 19.876H17.4609V17.0732H16.7041C16.0756 17.0732 15.4754 16.8012 15.0352 16.3223C14.5953 15.8437 14.3496 15.1971 14.3496 14.5254C14.3497 13.8537 14.5954 13.207 15.0352 12.7285C15.4754 12.2497 16.0757 11.9785 16.7041 11.9785H17.4609V11.4229C17.461 11.2749 17.5149 11.1306 17.6143 11.0225C17.7142 10.9138 17.8528 10.8496 18 10.8496ZM18.5391 19.876H19.2959C19.63 19.876 19.9539 19.7313 20.1943 19.4697C20.4351 19.2078 20.5722 18.85 20.5723 18.4746C20.5723 18.0991 20.4352 17.7405 20.1943 17.4785C19.9539 17.217 19.63 17.0732 19.2959 17.0732H18.5391V19.876ZM16.7041 13.124C16.37 13.124 16.0461 13.2687 15.8057 13.5303C15.5649 13.7922 15.4278 14.15 15.4277 14.5254C15.4277 14.9009 15.5648 15.2595 15.8057 15.5215C16.0461 15.783 16.37 15.9268 16.7041 15.9268H17.4609V13.124H16.7041Z"
{fill}
stroke={fill}
stroke-width="0.3"
/>
</g>
<defs>
<clipPath id="clip0_1_2">
<rect width="22" height="22" fill="white" />
</clipPath>
</defs>
</svg>
frontend/src/lib/components/runs/JobPreview.svelte
+16 -1
View File
@@ -15,8 +15,10 @@
import WorkflowTimeline from '../WorkflowTimeline.svelte'
import Popover from '../Popover.svelte'
import { isFlowPreview, isScriptPreview, truncateRev } from '$lib/utils'
import { createEventDispatcher, untrack } from 'svelte'
import { createEventDispatcher, setContext, untrack } from 'svelte'
import { ListFilter } from 'lucide-svelte'
import FlowAssetsHandler, { initFlowGraphAssetsCtx } from '../flows/FlowAssetsHandler.svelte'
import JobAssetsViewer from '../assets/JobAssetsViewer.svelte'
interface Props {
id: string
@@ -46,6 +48,11 @@
let viewTab = $state('result')
setContext(
'FlowGraphAssetContext',
initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] })
)
function asWorkflowStatus(x: any): Record<string, WorkflowStatus> {
return x as Record<string, WorkflowStatus>
}
@@ -192,6 +199,7 @@
<Tabs bind:selected={viewTab}>
<Tab size="xs" value="result">Result</Tab>
<Tab size="xs" value="logs">Logs</Tab>
<Tab size="xs" value="assets">Assets</Tab>
{#if isScriptPreview(job?.job_kind)}
<Tab size="xs" value="code">Code</Tab>
{/if}
@@ -205,6 +213,8 @@
<FlowStatusViewer jobId={job.id} workspaceId={job.workspace_id} />
</div>
</div>
{:else if viewTab == 'assets'}
<JobAssetsViewer {job} />
{:else}
<div class="flex flex-col border rounded-md p-2 mt-2 h-full overflow-auto">
{#if viewTab == 'logs'}
@@ -263,3 +273,8 @@
</div>
{/if}
</div>
<FlowAssetsHandler
modules={job?.raw_flow?.modules ?? []}
enableDbExplore
enablePathScriptAndFlowAssets
/>
frontend/src/lib/components/script_builder.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import type { NewScript } from '$lib/gen'
import type { AssetWithAccessType } from './assets/lib'
import type { AssetWithAltAccessType } from './assets/lib'
import type { ScriptBuilderWhitelabelCustomUi } from './custom_ui'
import type { DiffDrawerI } from './diff_drawer'
import type { ScriptBuilderFunctionExports } from './scriptBuilder'
@@ -9,7 +9,7 @@ import type { NewScriptWithDraftAndDraftTriggers, Trigger } from './triggers/uti
export interface ScriptBuilderProps {
script: NewScript & {
draft_triggers?: Trigger[]
fallback_access_types?: AssetWithAccessType[]
assets?: AssetWithAltAccessType[]
}
disableAi?: boolean
fullyLoaded?: boolean
frontend/src/lib/svelte5Utils.svelte.ts
+24 -16
View File
@@ -1,5 +1,6 @@
// https://github.com/sveltejs/svelte/issues/14600
import { untrack } from 'svelte'
import type { StateStore } from './utils'
export function withProps<Component, Props>(component: Component, props: Props) {
@@ -10,6 +11,11 @@ export function withProps<Component, Props>(component: Component, props: Props)
return ret
}
export function createState<T>(initialValue: T): T {
let s = $state(initialValue)
return s
}
export function stateSnapshot<T>(state: T) {
return $state.snapshot(state)
}
@@ -38,23 +44,25 @@ export function usePromise<T>(
status: 'loading',
__promise: undefined,
refresh: () => {
let promise = createPromise()
ret.__promise = promise
ret.status = 'loading'
if (clearValueOnRefresh) ret.value = undefined
ret.error = undefined
untrack(() => {
let promise = createPromise()
ret.__promise = promise
ret.status = 'loading'
if (clearValueOnRefresh) ret.value = undefined
ret.error = undefined
promise
.then((value) => {
if (ret.__promise !== promise) return
ret.value = value
ret.status = 'ok'
})
.catch((error) => {
if (ret.__promise !== promise) return
ret.error = error
ret.status = 'error'
})
promise
.then((value) => {
if (ret.__promise !== promise) return
ret.value = value
ret.status = 'ok'
})
.catch((error) => {
if (ret.__promise !== promise) return
ret.error = error
ret.status = 'error'
})
})
}
})
if (loadInit) ret.refresh()
frontend/src/lib/utils.ts
+17
View File
@@ -1465,3 +1465,20 @@ export function isS3Uri(uri: string): uri is S3Uri {
const match = uri.match(/^s3:\/\/([^/]*)\/(.*)$/)
return !!match && match.length === 3
}
export function uniqueBy<T>(array: T[], key: (t: T) => any): T[] {
const seen = new Set()
return array.filter((item) => {
const value = key(item)
if (seen.has(value)) {
return false
} else {
seen.add(value)
return true
}
})
}
export function pruneNullishArrayWithSet<T>(array: (T | null | undefined)[]): T[] {
return array.filter((item): item is T => item !== null && item !== undefined)
}
frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte
+14
View File
@@ -60,6 +60,9 @@
import { setContext } from 'svelte'
import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte'
import { Triggers } from '$lib/components/triggers/triggers.svelte'
import FlowAssetsHandler, {
initFlowGraphAssetsCtx
} from '$lib/components/flows/FlowAssetsHandler.svelte'
let flow: Flow | undefined = $state()
let can_write = false
@@ -91,6 +94,11 @@
triggersState
})
setContext(
'FlowGraphAssetContext',
initFlowGraphAssetsCtx({ getModules: () => flow?.value.modules ?? [] })
)
let previousPath: string | undefined = $state(undefined)
async function archiveFlow(): Promise<void> {
@@ -648,3 +656,9 @@
{/snippet}
</DetailPageLayout>
{/if}
<FlowAssetsHandler
modules={flow?.value.modules ?? []}
enableDbExplore
enablePathScriptAndFlowAssets
/>
frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte
+23 -2
View File
@@ -90,8 +90,12 @@
import CustomPopover from '$lib/components/CustomPopover.svelte'
import { isWindmillTooBigObject } from '$lib/components/job_args'
import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte'
import { untrack } from 'svelte'
import { setContext, untrack } from 'svelte'
import WorkerHostname from '$lib/components/WorkerHostname.svelte'
import FlowAssetsHandler, {
initFlowGraphAssetsCtx
} from '$lib/components/flows/FlowAssetsHandler.svelte'
import JobAssetsViewer from '$lib/components/assets/JobAssetsViewer.svelte'
let job: Job | undefined = $state()
let jobUpdateLastFetch: Date | undefined = $state()
@@ -99,7 +103,7 @@
let scriptProgress: number | undefined = $state(undefined)
let currentJobIsLongRunning: boolean = $state(false)
let viewTab: 'result' | 'logs' | 'code' | 'stats' = $state('result')
let viewTab: 'result' | 'logs' | 'code' | 'stats' | 'assets' = $state('result')
let selectedJobStep: string | undefined = $state(undefined)
let branchOrIterationN: number = $state(0)
@@ -118,6 +122,12 @@
let lastJobId: string | undefined = $state(undefined)
let concurrencyKey: string | undefined = $state(undefined)
setContext(
'FlowGraphAssetContext',
initFlowGraphAssetsCtx({ getModules: () => job?.raw_flow?.modules ?? [] })
)
async function getConcurrencyKey(job: Job | undefined) {
if (!job) return
lastJobId = job.id
@@ -925,6 +935,7 @@
<Tab value="result">Result</Tab>
<Tab value="logs">Logs</Tab>
<Tab value="stats">Metrics</Tab>
<Tab value="assets">Assets</Tab>
{#if isScriptPreview(job?.job_kind)}
<Tab value="code">Code</Tab>
{/if}
@@ -944,6 +955,10 @@
tag={job?.tag}
/>
</div>
{:else if viewTab == 'assets'}
<div class="w-full">
<JobAssetsViewer {job} />
</div>
{:else if viewTab == 'code'}
{#if job && 'raw_code' in job && job.raw_code}
<div class="text-xs">
@@ -1001,3 +1016,9 @@
{/if}
</div>
{/if}
<FlowAssetsHandler
modules={job?.raw_flow?.modules ?? []}
enableDbExplore
enablePathScriptAndFlowAssets
/>
openflow.openapi.yaml
+5 -5
View File
@@ -270,7 +270,7 @@ components:
type: string
is_trigger:
type: boolean
asset_fallback_access_types:
assets:
type: array
items:
type: object
@@ -287,10 +287,10 @@ components:
- resource
access_type:
type: string
enum:
- r
- w
- rw
enum: [r, w, rw]
alt_access_type:
type: string
enum: [r, w, rw]
required:
- type
- content