From f84c4cf9630ec49d3aac6f1195318aaaa002ea81 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 16 Jun 2026 17:21:23 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20pipeline=20Activity=20panel=20grouping,?= =?UTF-8?q?=20run=E2=86=94graph=20highlight,=20deploy-conflict=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Activity panel (view mode): - Group cascade runs by the connected component of the asset-dispatch graph (new GET /jobs/asset_dispatch_edges over the dispatch_event table, incl. join_pending inputs), headed by the earliest originating run + its trigger, with a "+N" chip for joins fed by multiple triggers. - Success/failure count histogram with drag-to-filter brushing, an always-on time axis + per-bar tooltips, a Reset, and Last hour/24h/48h/7/30/90d ranges. - Node run-count/status badges now derive from the same merged historic+live events the panel shows (previously session-only). Run ↔ graph highlight: - Hovering a run row (or a group header → the whole cascade) rings the node(s), animates their incident edges, and borders the adjacent assets in the edge hue (blue write / gray read); expanding a run pins a soft-blue ring. - Switching edit→view re-surfaces the Activity feed. Deploy: - Live-content autosave for the open pipeline draft + an autosave indicator. - Re-saving a script now chains off the hash just created instead of a stale parent_hash (fixes the "lineage must be linear" error on a second save), and a genuine concurrent deploy opens a keep-mine / view-latest conflict modal. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...3460b6438d0a320b5554acec0beddbdeb008b.json | 74 ++++ backend/windmill-api/openapi.yaml | 72 ++++ backend/windmill-api/src/jobs.rs | 81 +++++ .../AssetGraph/ActivityHistogram.svelte | 155 ++++++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 84 ++++- .../AssetGraph/AssetGraphDetailsPane.svelte | 130 ++++++- .../AssetGraph/PipelineActivityPanel.svelte | 331 +++++++++++++++++- .../AssetGraph/pipelineHistory.svelte.ts | 34 ++ .../(logged)/pipeline/[folder]/+page.svelte | 128 ++++++- 9 files changed, 1056 insertions(+), 33 deletions(-) create mode 100644 backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json create mode 100644 frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte diff --git a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json new file mode 100644 index 0000000000..24d70efae1 --- /dev/null +++ b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n de.producer_job_id AS \"producer_job_id!\",\n de.child_job_id,\n de.subscriber_path AS \"subscriber_path!\",\n de.outcome::text AS \"outcome!\",\n de.asset_kind AS \"asset_kind!: windmill_common::assets::AssetKind\",\n de.asset_path AS \"asset_path!\",\n de.created_at AS \"created_at!\"\n FROM dispatch_event de\n JOIN v2_job pj ON pj.id = de.producer_job_id\n WHERE de.workspace_id = $1\n AND de.outcome IN ('dispatched', 'join_pending')\n AND de.subscriber_path LIKE $2\n AND ($3::timestamptz IS NULL OR de.created_at >= $3)\n ORDER BY de.created_at DESC, de.id DESC\n LIMIT 4000", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "producer_job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "child_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "subscriber_path!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "outcome!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "asset_kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 5, + "name": "asset_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "created_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [ + false, + true, + false, + null, + false, + false, + false + ] + }, + "hash": "8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8cc9efd2ef..e99a1917dc 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13006,6 +13006,78 @@ paths: - outcome - created_at + /w/{workspace}/jobs/asset_dispatch_edges: + get: + summary: list asset-cascade producer→child job edges for a folder + description: > + Returns the `dispatched` asset-trigger edges (producer job → child job) + whose subscriber lives under `path_start`. Lets a pipeline view + reconstruct the cascade tree of a folder by job id and group connected + runs. Visibility follows the producer job's RLS. + operationId: listAssetDispatchEdges + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path_start + in: query + required: true + description: Folder path prefix the children live under, e.g. `f/orders/`. + schema: + type: string + - name: created_after + in: query + required: false + description: Only edges dispatched at/after this instant. + schema: + type: string + format: date-time + responses: + "200": + description: asset-cascade edges for the folder + content: + application/json: + schema: + type: array + items: + type: object + properties: + producer_job_id: + type: string + format: uuid + child_job_id: + type: string + format: uuid + description: Set for `dispatched`; absent for `join_pending` inputs. + subscriber_path: + type: string + outcome: + type: string + enum: + - dispatched + - join_pending + asset_kind: + type: string + enum: + - s3object + - resource + - variable + - ducklake + - datatable + - volume + asset_path: + type: string + created_at: + type: string + format: date-time + required: + - producer_job_id + - subscriber_path + - outcome + - asset_kind + - asset_path + - created_at + /w/{workspace}/jobs/completed/delete/{id}: post: summary: delete completed job (erase content but keep run id) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a9e307fc6e..d10abdf43e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -270,6 +270,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) + .route("/asset_dispatch_edges", get(list_asset_dispatch_edges)) .route( "/list_selected_job_groups", // We use post because sending a huge array as a query param can produce @@ -9145,6 +9146,86 @@ async fn get_dispatch_events( )) } +/// One asset-cascade dispatch record, for reconstructing the cascade graph of a +/// pipeline folder in the Activity panel. `dispatched` rows carry the resolved +/// `child_job_id` (a real producer→child job edge); `join_pending` rows are the +/// pre-completion inputs of an AND-join (no child yet) — the client links them +/// to the eventual child of the same `subscriber_path` so a join's separate +/// trigger chains merge into one group. `skipped` is omitted. +#[derive(Serialize)] +struct AssetDispatchEdge { + producer_job_id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + child_job_id: Option, + subscriber_path: String, + outcome: String, + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + created_at: chrono::DateTime, +} + +#[derive(Deserialize)] +struct AssetDispatchEdgesQuery { + /// Folder path prefix the children live under, e.g. `f/orders/`. Matched + /// against `subscriber_path` — every intra-pipeline cascade edge has its + /// child in the folder, so this captures the whole folder's cascades. + path_start: String, + /// Only edges dispatched at/after this instant (align with the activity + /// window the client already loaded). Omit for the default cap. + created_after: Option>, +} + +/// Asset-cascade edges for a pipeline folder. RLS on the joined `v2_job` +/// producer row limits this to cascades whose producer the caller can already +/// see (same visibility as the folder's job list). +async fn list_asset_dispatch_edges( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult> { + let like = format!("{}%", query.path_start); + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#"SELECT + de.producer_job_id AS "producer_job_id!", + de.child_job_id, + de.subscriber_path AS "subscriber_path!", + de.outcome::text AS "outcome!", + de.asset_kind AS "asset_kind!: windmill_common::assets::AssetKind", + de.asset_path AS "asset_path!", + de.created_at AS "created_at!" + FROM dispatch_event de + JOIN v2_job pj ON pj.id = de.producer_job_id + WHERE de.workspace_id = $1 + AND de.outcome IN ('dispatched', 'join_pending') + AND de.subscriber_path LIKE $2 + AND ($3::timestamptz IS NULL OR de.created_at >= $3) + ORDER BY de.created_at DESC, de.id DESC + LIMIT 4000"#, + &w_id, + like, + query.created_after, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| AssetDispatchEdge { + producer_job_id: r.producer_job_id, + child_job_id: r.child_job_id, + subscriber_path: r.subscriber_path, + outcome: r.outcome, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + created_at: r.created_at, + }) + .collect(), + )) +} + async fn get_completed_job_timing( OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, diff --git a/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte new file mode 100644 index 0000000000..fcfd95a5d3 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte @@ -0,0 +1,155 @@ + + +
+ + +
+ {fmtTime(from)} + now +
+
diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index a46c73f3c1..cd73221626 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -125,6 +125,13 @@ // form (with the auto-generated S3 picker) for the given script. // data_upload has no trigger row; it's a UI-first entry point. onOpenDataUpload?: (scriptPath: string) => void + // Script paths the cursor is over in the Activity panel — a thin neutral + // ring each. One entry for a single run row; the whole cascade's runs + // when hovering a group header. Distinct from `selectedRunPaths`. + hoveredPaths?: string[] + // Script paths of the expanded (pinned) run(s) — a soft blue ring that + // persists until collapsed. + selectedRunPaths?: string[] } let { graph, @@ -143,9 +150,18 @@ onEditTrigger, onDeleteTrigger, onOpenWebhook, - onOpenDataUpload + onOpenDataUpload, + hoveredPaths, + selectedRunPaths }: Props = $props() + // `${kind}:${path}` ids for the hovered / pinned runs (both script and flow + // variants, since the run row's kind isn't known here). + const runIds = (paths: string[] | undefined): Set => + new Set((paths ?? []).flatMap((p) => [`script:${p}`, `flow:${p}`])) + let hoveredRunIdSet = $derived(runIds(hoveredPaths)) + let selectedRunIdSet = $derived(runIds(selectedRunPaths)) + const ADD_NODE_ID = '__add__' type BuiltEdge = { @@ -532,11 +548,25 @@ // Compensate for the + node being narrower than its layout slot // so it visually centers over the node(s) below. const xShift = n.id === ADD_NODE_ID ? (NODE.width - ADD_NODE_WIDTH) / 2 : 0 + // Activity-panel emphasis (purely visual rings, kept off `selected` + // which swaps the details pane). Hover wins over pin so the cursor + // always tracks. + const runClass = hoveredRunIdSet.has(n.id) + ? 'wm-run-hover' + : selectedRunIdSet.has(n.id) + ? 'wm-run-selected' + : undefined + // Asset border matching the connecting edge's hue (only when no run + // class applies — a node is either a runnable or an asset). + const assetEmph = assetEmphasis.get(n.id) + const assetClass = + assetEmph === 'output' ? 'wm-asset-output' : assetEmph === 'input' ? 'wm-asset-input' : undefined return { id: n.id, type: n.type, position: { x: p.x + xCenter + xShift, y: p.y + 40 }, data: n.data, + class: runClass ?? assetClass, selected: n.id === selectedId, // All nodes non-draggable: the layout is sugiyama-computed, // dragging would fight the reactive re-layout. Selection is @@ -562,6 +592,29 @@ ...(activeRunnableNodeId ? [activeRunnableNodeId] : []) ]) ) + // Node ids emphasized from the Activity panel (hovered or pinned runs). + // Their incident edges get the same flow animation as a live run, so + // hovering a run (or a whole group) traces its data path through the graph. + let emphasizedRunIdSet = $derived(new Set([...hoveredRunIdSet, ...selectedRunIdSet])) + // Asset nodes adjacent to an emphasized run, tagged by edge family so their + // border matches the edge hue: `output` (a write, blue) / `input` (a read or + // asset-trigger, gray). Output wins if an asset is both. + let assetEmphasis = $derived.by>(() => { + const m = new Map() + if (emphasizedRunIdSet.size === 0) return m + for (const e of model.edges) { + if (e.kind === 'lineage-write' && emphasizedRunIdSet.has(e.source)) { + m.set(e.target, 'output') + } else if ( + (e.kind === 'lineage-read' || e.kind === 'trigger-asset') && + emphasizedRunIdSet.has(e.target) && + m.get(e.source) !== 'output' + ) { + m.set(e.source, 'input') + } + } + return m + }) let flowEdges = $derived.by(() => model.edges // Anchor edges are layout-only. @@ -570,6 +623,12 @@ const touchesActiveRunnable = activeRunnableIdSet.size > 0 && (activeRunnableIdSet.has(e.source) || activeRunnableIdSet.has(e.target)) + // Edge incident to an Activity-panel hovered/pinned run — animate + // it too so the row↔graph link is unmistakable. + const touchesEmphasis = + emphasizedRunIdSet.size > 0 && + (emphasizedRunIdSet.has(e.source) || emphasizedRunIdSet.has(e.target)) + const flowAnimated = touchesActiveRunnable || touchesEmphasis let style: string let animated = false let markerColor: string | undefined = undefined @@ -586,16 +645,16 @@ switch (e.kind) { case 'lineage-write': style = 'stroke: rgb(59 130 246); stroke-width: 2px;' - animated = touchesActiveRunnable + animated = flowAnimated markerColor = 'rgb(59 130 246)' break case 'lineage-read': style = 'stroke: rgb(156 163 175); stroke-width: 1.25px;' - animated = touchesActiveRunnable + animated = flowAnimated break case 'trigger-asset': style = 'stroke: rgb(107 114 128); stroke-width: 2px;' - animated = touchesActiveRunnable + animated = flowAnimated markerColor = 'rgb(107 114 128)' label = 'triggers' labelStyle = 'fill: rgb(107 114 128); font-size: 10px; font-weight: 600;' @@ -743,4 +802,21 @@ :global(.svelte-flow__node.selected .drop-shadow-sm) { @apply outline outline-2 outline-blue-500; } + /* Activity-panel emphasis — soft, monochromatic, less prominent than the + blue details selection above. Hover is a thin neutral ring (transient); + pinning an expanded run is a soft-blue ring. */ + :global(.svelte-flow__node.wm-run-hover .drop-shadow-sm) { + @apply outline outline-1 outline-gray-400 dark:outline-gray-500; + } + :global(.svelte-flow__node.wm-run-selected .drop-shadow-sm) { + @apply outline outline-2 outline-blue-400/70; + } + /* Assets adjacent to an emphasized run — border hue matches the edge: + blue for a write (output), gray for a read / asset-trigger (input). */ + :global(.svelte-flow__node.wm-asset-output .drop-shadow-sm) { + @apply outline outline-2 outline-blue-400/70; + } + :global(.svelte-flow__node.wm-asset-input .drop-shadow-sm) { + @apply outline outline-2 outline-gray-400; + } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index df04960f94..71d4f85921 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -71,6 +71,12 @@ // edited past the seeded template. Fires on every keystroke that // changes the inferred set. onAssetsChange?: (scriptPath: string | undefined, assets: AssetWithAltAccessType[]) => void + // Emits the live editor buffer on every keystroke so the parent can + // autosave the in-flight content WITHOUT waiting for the pane teardown + // (`onDraftPersist`). `onDraftPersist` stays the authoritative commit on + // navigate-away; this is the continuous feed that keeps the draft's + // server copy current while the user is still typing. + onContentChange?: (scriptPath: string | undefined, content: string) => void // Fires when the user navigates away from a draft (selects another // node, closes the pane, etc.) — the parent must persist the new // content + write outputs into its drafts Map. Without this, @@ -196,6 +202,7 @@ onDiscard, onAnnotationsChange, onAssetsChange, + onContentChange, onDraftPersist, onScriptRenamed, onScriptRemoved, @@ -539,6 +546,10 @@ if (readOnly) return onAssetsChange?.(script?.path, liveBodyAssets ?? []) }) + $effect(() => { + if (readOnly) return + onContentChange?.(script?.path, script?.content ?? '') + }) async function save() { if (!script) return @@ -552,7 +563,7 @@ } catch { sendUserToast(`Could not parse code, are you sure it is valid?`, true) } - await ScriptService.createScript({ + const newHash = await ScriptService.createScript({ workspace, requestBody: { ...script, @@ -579,6 +590,11 @@ assets: (liveBodyAssets ?? []) as any } }) + // Chain the NEXT save off the version we just created. Without this + // a second save re-sends the now-stale `parent_hash`, which the + // backend rejects as a lineage fork ("no 2 scripts can have the + // same parent"). + if (typeof newHash === 'string' && newHash) script.hash = newHash sendUserToast(`Saved ${script.path}`) if (isDraft) { onDraftSaved?.(script.path) @@ -587,12 +603,56 @@ onPersistedSaved?.(script.path) } } catch (e: any) { - sendUserToast(`Save failed: ${e?.body ?? e?.message ?? e}`, true) + const msg = String(e?.body ?? e?.message ?? e) + // The deployed head moved off our `parent_hash` since we opened it — + // someone else (or another tab) deployed in between. Offer an + // explicit resolution instead of a cryptic lineage error. + if (/lineage must be linear|same parent_hash/i.test(msg)) { + conflictOpen = true + } else { + sendUserToast(`Save failed: ${msg}`, true) + } } finally { saving = false } } + // --- Deploy conflict resolution (another deploy landed since we opened) --- + let conflictOpen = $state(false) + let resolvingConflict = $state(false) + async function overwriteWithMine() { + if (!script) return + resolvingConflict = true + try { + // Re-base onto the current head so the retry is a linear deploy on + // top of theirs (our content wins). + const latest = await ScriptService.getScriptByPath({ workspace, path: script.path }) + script.hash = String(latest.hash) + conflictOpen = false + await save() + } catch (e: any) { + sendUserToast(`Could not resolve conflict: ${e?.body ?? e?.message ?? e}`, true) + } finally { + resolvingConflict = false + } + } + async function viewLatestVersion() { + if (!script) return + resolvingConflict = true + try { + const latest = await ScriptService.getScriptByPath({ workspace, path: script.path }) + // Replace the editor buffer with the latest deployed version. The + // user's unsaved edits are dropped — surfaced clearly in the modal. + script = structuredClone($state.snapshot(latest) as Script) + conflictOpen = false + sendUserToast('Loaded the latest deployed version') + } catch (e: any) { + sendUserToast(`Could not load the latest version: ${e?.body ?? e?.message ?? e}`, true) + } finally { + resolvingConflict = false + } + } + let isScriptView = $derived( isDraft || (selection?.kind === 'runnable' && selection.runnable_kind === 'script') ) @@ -1121,6 +1181,72 @@ {/if} +{#if conflictOpen} + + +{/if} +