From dcbd77b796b4eafad9a0759ac2e5383e9b891d9f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 19 Apr 2026 03:10:09 +0000 Subject: [PATCH] feat: add workspace asset graph view Workspace-wide canvas of assets and their producer/consumer scripts, reachable from the assets page. Left-to-right layered layout via d3-dag sugiyama, rendered with @xyflow/svelte (same stack as the flow editor). GET /w/:ws/assets/graph returns deduped nodes + edges. Follow-ups: filters (kind/folder/search), node detail drawer, inline script edit from a clicked node. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...87562dbdac633f3248ac228e00b7c8b49b800.json | 94 +++++++++++ backend/windmill-api-assets/src/lib.rs | 119 ++++++++++++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 146 ++++++++++++++++++ .../assets/AssetGraph/AssetNode.svelte | 42 +++++ .../assets/AssetGraph/RunnableNode.svelte | 42 +++++ .../assets/AssetGraph/assetGraphLayout.ts | 77 +++++++++ .../lib/components/assets/AssetGraph/types.ts | 39 +++++ .../(root)/(logged)/assets/+page.svelte | 16 +- .../(root)/(logged)/assets/graph/+page.js | 5 + .../(root)/(logged)/assets/graph/+page.svelte | 105 +++++++++++++ 10 files changed, 683 insertions(+), 2 deletions(-) create mode 100644 backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/types.ts create mode 100644 frontend/src/routes/(root)/(logged)/assets/graph/+page.js create mode 100644 frontend/src/routes/(root)/(logged)/assets/graph/+page.svelte diff --git a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json new file mode 100644 index 0000000000..c126b6371a --- /dev/null +++ b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json @@ -0,0 +1,94 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n asset.kind AS \"asset_kind!: AssetKind\",\n asset.path AS \"asset_path!\",\n asset.usage_kind AS \"usage_kind!: AssetUsageKind\",\n asset.usage_path AS \"usage_path!\",\n asset.usage_access_type::text AS \"access_type\"\n FROM asset\n WHERE asset.workspace_id = $1\n AND asset.usage_kind IN ('script', 'flow')\n AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2))\n AND ($3::text IS NULL OR asset.usage_path LIKE $3)\n GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "usage_kind!: AssetUsageKind", + "type_info": { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "usage_path!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "access_type", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800" +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 0c25cd243d..5f5b287dc3 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -20,6 +20,7 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_assets)) .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) + .route("/graph", get(asset_graph)) } #[derive(Deserialize)] @@ -363,3 +364,121 @@ async fn list_favorites( Ok(Json(favorites)) } + +// ------------------------------------------------------------------ +// GET /w/:workspace/assets/graph +// ------------------------------------------------------------------ +// Workspace-wide asset ↔ runnable graph. One row per unique +// (asset_kind, asset_path, usage_kind, usage_path, access_type) — the +// frontend aggregates into nodes and edges. + +#[derive(Deserialize)] +struct GraphQuery { + pub asset_kinds: Option, + pub folder: Option, +} + +#[derive(Serialize, Debug)] +struct GraphAssetNode { + kind: AssetKind, + path: String, +} + +#[derive(Serialize, Debug)] +struct GraphRunnableNode { + path: String, + usage_kind: AssetUsageKind, +} + +#[derive(Serialize, Debug)] +struct GraphEdge { + runnable_path: String, + runnable_kind: AssetUsageKind, + asset_kind: AssetKind, + asset_path: String, + access_type: Option, +} + +#[derive(Serialize, Debug)] +struct AssetGraphResponse { + assets: Vec, + runnables: Vec, + edges: Vec, +} + +async fn asset_graph( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + let kind_filter: Option> = q.asset_kinds.as_ref().map(|s| { + s.split(',') + .filter_map(|k| { + serde_json::from_value::(Value::String(k.trim().into())).ok() + }) + .collect() + }); + let kind_filter_ref = kind_filter.as_deref(); + + let folder_filter = q.folder.as_deref().map(|f| format!("f/{}/%", f)); + + // One row per (asset_kind, asset_path, usage_kind, usage_path, access_type). + // The `usage_kind IN ('script','flow')` clause excludes `job`-kind usage rows + // (runtime-detected, ephemeral) so the graph stays stable. + let rows = sqlx::query!( + r#" + SELECT + asset.kind AS "asset_kind!: AssetKind", + asset.path AS "asset_path!", + asset.usage_kind AS "usage_kind!: AssetUsageKind", + asset.usage_path AS "usage_path!", + asset.usage_access_type::text AS "access_type" + FROM asset + WHERE asset.workspace_id = $1 + AND asset.usage_kind IN ('script', 'flow') + AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2)) + AND ($3::text IS NULL OR asset.usage_path LIKE $3) + GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type + "#, + &w_id, + kind_filter_ref as Option<&[AssetKind]>, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + let mut edges = Vec::with_capacity(rows.len()); + let mut asset_set: std::collections::HashSet<(AssetKind, String)> = Default::default(); + let mut runnable_set: std::collections::HashSet<(AssetUsageKind, String)> = Default::default(); + + for r in rows { + asset_set.insert((r.asset_kind, r.asset_path.clone())); + runnable_set.insert((r.usage_kind, r.usage_path.clone())); + edges.push(GraphEdge { + runnable_path: r.usage_path, + runnable_kind: r.usage_kind, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + access_type: r.access_type, + }); + } + + let mut assets: Vec = asset_set + .into_iter() + .map(|(kind, path)| GraphAssetNode { kind, path }) + .collect(); + assets.sort_by(|a, b| a.path.cmp(&b.path)); + + let mut runnables: Vec = runnable_set + .into_iter() + .map(|(usage_kind, path)| GraphRunnableNode { path, usage_kind }) + .collect(); + runnables.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(Json(AssetGraphResponse { assets, runnables, edges })) +} diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte new file mode 100644 index 0000000000..7a233a247c --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -0,0 +1,146 @@ + + +
+ + + + + +
+ + diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte new file mode 100644 index 0000000000..e4151f7cba --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -0,0 +1,42 @@ + + +
+ +
+ +
+ {data.asset_kind} + {data.path} +
+
+ +
diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte new file mode 100644 index 0000000000..4a649429e0 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -0,0 +1,42 @@ + + +
+ +
+ +
+ {label} + {data.path} +
+ e.stopPropagation()} + aria-label="Open {label.toLowerCase()} editor" + > + + +
+ +
diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts new file mode 100644 index 0000000000..6a9df0539e --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts @@ -0,0 +1,77 @@ +import { dagStratify, sugiyama, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' +import type { AssetGraphNodeData } from './types' + +const NODE_WIDTH = 260 +const NODE_HEIGHT = 64 +const LAYER_GAP = 80 +const SIBLING_GAP = 32 + +interface GraphInput { + nodes: Array<{ id: string; data: AssetGraphNodeData }> + edges: Array<{ source: string; target: string }> +} + +interface Positioned { + x: number + y: number +} + +// Sugiyama layered layout: producers left → assets middle → consumers right. +// Falls back to a stable grid if d3-dag throws (e.g., on cyclic inputs — +// shouldn't happen in practice for asset usage). +export function layoutAssetGraph(graph: GraphInput): Map { + const byId = new Map() + if (graph.nodes.length === 0) return byId + + const parentsByChild = new Map() + for (const n of graph.nodes) parentsByChild.set(n.id, []) + for (const e of graph.edges) { + const arr = parentsByChild.get(e.target) + if (arr && arr.indexOf(e.source) === -1) arr.push(e.source) + } + + try { + const dagNodes = graph.nodes.map((n) => ({ + id: n.id, + parentIds: parentsByChild.get(n.id) ?? [] + })) + const dag = dagStratify().id(({ id }: { id: string }) => id)(dagNodes) + const layout = sugiyama() + .decross(graph.nodes.length > 30 ? decrossTwoLayer() : decrossOpt()) + .coord(coordCenter()) + .nodeSize( + () => [NODE_WIDTH + SIBLING_GAP, NODE_HEIGHT + LAYER_GAP] as readonly [number, number] + ) + layout(dag as any) + for (const desc of dag.descendants()) { + const id = (desc as any).data.id as string + byId.set(id, { + x: (desc as any).x ?? 0, + y: ((desc as any).y ?? 0) - (NODE_HEIGHT + LAYER_GAP) / 2 + }) + } + // Normalize so min x,y = 0 + let minX = Infinity + let minY = Infinity + for (const p of byId.values()) { + if (p.x < minX) minX = p.x + if (p.y < minY) minY = p.y + } + if (isFinite(minX) && isFinite(minY)) { + for (const p of byId.values()) { + p.x -= minX + p.y -= minY + } + } + return byId + } catch { + const cols = Math.max(1, Math.ceil(Math.sqrt(graph.nodes.length))) + graph.nodes.forEach((n, i) => { + byId.set(n.id, { + x: (i % cols) * (NODE_WIDTH + SIBLING_GAP), + y: Math.floor(i / cols) * (NODE_HEIGHT + LAYER_GAP) + }) + }) + return byId + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts new file mode 100644 index 0000000000..70447eb85a --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -0,0 +1,39 @@ +import type { AssetKind } from '$lib/gen' + +export type GraphUsageKind = 'script' | 'flow' + +export interface AssetGraphAssetNode { + kind: AssetKind + path: string +} + +export interface AssetGraphRunnableNode { + path: string + usage_kind: GraphUsageKind +} + +export interface AssetGraphEdge { + runnable_path: string + runnable_kind: GraphUsageKind + asset_kind: AssetKind + asset_path: string + access_type: 'r' | 'w' | 'rw' | null +} + +export interface AssetGraphResponse { + assets: AssetGraphAssetNode[] + runnables: AssetGraphRunnableNode[] + edges: AssetGraphEdge[] +} + +export type AssetGraphNodeData = + | { + kind: 'asset' + asset_kind: AssetKind + path: string + } + | { + kind: 'runnable' + runnable_kind: GraphUsageKind + path: string + } diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/+page.svelte index 50ed6f041b..ea7af59a6d 100644 --- a/frontend/src/routes/(root)/(logged)/assets/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/assets/+page.svelte @@ -34,7 +34,8 @@ import { untrack } from 'svelte' import { VolumeService } from '$lib/gen' import VolumesDrawer from '$lib/components/assets/VolumesDrawer.svelte' - import { HardDriveIcon } from 'lucide-svelte' + import { HardDriveIcon, NetworkIcon } from 'lucide-svelte' + import { base } from '$lib/base' interface AssetCursor { created_at?: string @@ -155,7 +156,18 @@ title="Assets" tooltip="Assets show up here whenever you use them in Windmill." documentationLink="https://www.windmill.dev/docs/core_concepts/assets" - /> + > +
+ +
+
diff --git a/frontend/src/routes/(root)/(logged)/assets/graph/+page.js b/frontend/src/routes/(root)/(logged)/assets/graph/+page.js new file mode 100644 index 0000000000..99dfd42c32 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/assets/graph/+page.js @@ -0,0 +1,5 @@ +export function load() { + return { + stuff: { title: 'Asset Graph' } + } +} diff --git a/frontend/src/routes/(root)/(logged)/assets/graph/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/graph/+page.svelte new file mode 100644 index 0000000000..72e32afdba --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/assets/graph/+page.svelte @@ -0,0 +1,105 @@ + + + + Asset graph — Windmill + + +{#if $userStore?.operator} +
Page not available for operators.
+{:else} +
+
+
+ +
+ +

Asset graph

+
+ + Workspace-wide view of assets and their producers/consumers. + +
+
+ {#if graph} + + {graph.assets.length} assets · {graph.runnables.length} runnables · {graph.edges.length} + edges + + {/if} + +
+
+ +
+ {#if loading} +
+ + Loading graph… +
+ {:else if error} +
+ Failed to load graph: {error} +
+ {:else if graph && graph.assets.length === 0 && graph.runnables.length === 0} +
+ No assets are referenced by scripts or flows in this workspace yet. +
+ {:else if graph} + + {/if} +
+
+{/if}