From 1710b8a77bb0fb569a70ecee2bf4c8fabbe55ebc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 1 May 2026 11:57:57 +0000 Subject: [PATCH] all --- .../src/lib/components/S3FilePreview.svelte | 17 +++- .../AssetGraph/AssetGraphDetailsPane.svelte | 65 +++++++++----- .../assets/AssetGraph/AssetNode.svelte | 21 +---- .../assets/AssetGraph/AssetRunsPanel.svelte | 65 +++++++++++--- .../assets/AssetGraph/DataTablePreview.svelte | 87 +++++++++++++++++++ .../(logged)/pipeline/[folder]/+page.svelte | 9 +- 6 files changed, 212 insertions(+), 52 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte diff --git a/frontend/src/lib/components/S3FilePreview.svelte b/frontend/src/lib/components/S3FilePreview.svelte index 6e3de40668..2a7c3977cd 100644 --- a/frontend/src/lib/components/S3FilePreview.svelte +++ b/frontend/src/lib/components/S3FilePreview.svelte @@ -37,6 +37,11 @@ // metadata out themselves (the picker already has its own). showMetadata?: boolean class?: string + // Bump this to force a re-fetch (metadata + preview). Used by the + // asset detail pane after an upstream producer run completes — + // without it, the "Asset not yet materialized" empty state stays + // pinned until the user re-selects the asset. + refreshKey?: any } let { @@ -45,7 +50,8 @@ loadFilePreviewRequest = HelpersService.loadFilePreview, loadFileMetadataRequest = HelpersService.loadFileMetadata, showMetadata = false, - class: className = '' + class: className = '', + refreshKey }: Props = $props() let csvSeparatorChar: string = $state(',') @@ -91,12 +97,15 @@ return body.includes('not found') || body.includes('404') } - // Reload whenever the file key changes. Tracking the workspace too — - // the asset graph spans workspaces so a re-mount with the same key but - // a different ws should refetch. + // Reload whenever the file key, workspace, or external refreshKey + // changes. The refreshKey path is what lets the asset pane re-check + // existence after an upstream run completes — moving from the + // "not yet materialized" empty state to the actual preview without + // requiring the user to re-click the asset. $effect(() => { const key = fileKey const ws = $workspaceStore + void refreshKey if (!key || !ws) { fileMetadata = undefined filePreview = undefined diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 777c8fba17..67719e94f2 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -23,6 +23,7 @@ import { parsePipelineAnnotations, type PipelineAnnotations } from './parsePipelineAnnotations' import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte' import S3FilePreview from '$lib/components/S3FilePreview.svelte' + import DataTablePreview from './DataTablePreview.svelte' import AssetRunsPanel from './AssetRunsPanel.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' import { fade } from 'svelte/transition' @@ -69,6 +70,9 @@ // re-fetches the listing immediately (rather than waiting on its // background poll tick). runsRefreshKey?: any + // Job id of the most recently dispatched run. Forwarded to the + // runs panel so that clicking play auto-selects the new run. + runsPendingJobId?: string | undefined } let { selection, @@ -81,9 +85,17 @@ onScriptRenamed, onScriptRemoved, selectionProducers = [], - runsRefreshKey + runsRefreshKey, + runsPendingJobId }: Props = $props() + // Bumped when the runs panel reports a watched job has reached a + // terminal state. Drives S3FilePreview's refreshKey so the preview + // re-checks existence after a producer run finishes — moves the + // "not yet materialized" empty state to the actual preview without + // requiring the user to re-click the asset. + let previewRefreshKey = $state(0) + // When `draftScript` is provided we bypass the fetch entirely and edit // it locally; saving calls ScriptService.createScript to deploy it. let scriptRes = resource( @@ -334,25 +346,38 @@
{#if selection?.kind === 'asset' && !isDraft} - {#if selection.asset_kind === 's3object'} - - - - - - - - - - {:else} -
- Asset details. Use the producer/consumer arrows in the graph to navigate. -
- {/if} + + + + {#if selection.asset_kind === 's3object'} + + {:else if selection.asset_kind === 'datatable'} + + {:else} +
+ No inline preview yet for {selection.asset_kind}. Use the producer/consumer arrows in + the graph to navigate. Runs of the upstream script are below. +
+ {/if} + + + (previewRefreshKey += 1)} + /> + + {:else if selection?.kind === 'runnable' && selection.runnable_kind === 'flow' && !isDraft}
Flows are not editable inline. Use the open-in-editor button above. diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index e3c7b354cd..359e073f9f 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -9,7 +9,6 @@ import type { ScriptLang } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/utils' - import { base } from '$lib/base' // Shape used for both the data prop and the run callback. Drafts carry // `content` / `language` so the page-level run handler can dispatch to @@ -74,22 +73,10 @@ // Fire every script producer in parallel — matches what would // happen if every upstream trigger fired together. The handler // internally dispatches to runScriptByPath / runScriptPreview - // based on producer.unsaved. - const jobs = (await Promise.all(scriptProducers.map((p) => handler(p)))).filter( - (j): j is string => !!j - ) - if (jobs.length === 1) { - sendUserToast(`Running ${scriptProducers[0].path}`, false, [ - { - label: 'Open run', - callback: () => { - window.open(`${base}/run/${jobs[0]}?workspace=${$workspaceStore}`, '_blank') - } - } - ]) - } else if (jobs.length > 1) { - sendUserToast(`Running ${jobs.length} producers of this asset`) - } + // based on producer.unsaved. No success toast — the runs panel + // auto-selects the new job and shows status/logs/output, so + // the toast was redundant. + await Promise.all(scriptProducers.map((p) => handler(p))) } catch (err: any) { sendUserToast(`Failed to run: ${err.body ?? err.message}`, true) } finally { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte index 2b92dd8a9d..a1a2f0d52d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte @@ -33,8 +33,19 @@ // uses it after dispatching a run so the new job appears without // waiting for the next poll tick. refreshKey?: any + // Most-recently-dispatched job id. When this changes, the panel + // switches selection to it so the user lands on their just-started + // run without an extra click. Independent of refreshKey so callers + // can refresh without forcing a selection change. + pendingJobId?: string | undefined + // Fires when the watched job transitions to terminal (success or + // failure). The asset detail pane uses this to re-check the + // asset's preview — a successful run materializes the asset, so + // "not yet materialized" can move to the actual preview without + // the user re-selecting. + onRunCompleted?: () => void } - let { producers, refreshKey }: Props = $props() + let { producers, refreshKey, pendingJobId, onRunCompleted }: Props = $props() let runnableProducers = $derived(producers.filter((p) => p.kind === 'script')) // Stable string key for the producer set. The parent re-derives @@ -98,20 +109,54 @@ } }) + // Track the last pendingJobId we honored. Without this guard, manually + // selecting a *different* run from the history popover would be undone + // by a re-run of this effect (e.g. on parent re-derivation) — every + // trip would slam selectedId back to pendingJobId. + let appliedPendingJobId = $state(undefined) + $effect(() => { + const id = pendingJobId + if (id && id !== appliedPendingJobId) { + appliedPendingJobId = id + selectedId = id + } + }) + + // Track the id we last started watching so we don't restart the + // JobLoader stream on every effect re-run. The earlier flood (~4400 + // `/jobs_u/get/` requests per second) came from JobLoader.watchJob + // being called on every effect tick: each call resets internal state + // and fires a fresh `getJob`, so a few extra effect runs per second + // snowballed into thousands of fetches. + let lastWatchedId = $state(undefined) + $effect(() => { - // Watch the selected job as soon as one is picked. const id = selectedId + if (id === lastWatchedId) return + lastWatchedId = id if (!id) { - selectedJob = undefined - void jobLoader?.clearCurrentJob?.() + // untrack: clearCurrentJob reads JobLoader internals (tracked + // state in another component); without untrack, those reads + // become deps of this effect and any change to them + // (e.g. JobLoader's own bind:isLoading writes) would re-run + // it. + untrack(() => { + selectedJob = undefined + void jobLoader?.clearCurrentJob?.() + }) return } - void jobLoader?.watchJob(id, { - done: () => { - // When a run we're watching finishes, refresh the listing so - // its row shows the terminal status. - void refresh() - } + untrack(() => { + void jobLoader?.watchJob(id, { + done: () => { + // When a run we're watching finishes, refresh the listing + // so its row shows the terminal status, and notify the + // parent so the asset preview can re-check existence — + // a successful run materializes the asset. + void refresh() + onRunCompleted?.() + } + }) }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte new file mode 100644 index 0000000000..3475f8236e --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte @@ -0,0 +1,87 @@ + + +
+ {#if !input} +
No datatable selected.
+ {:else if !parsed.table} +
+ Pick a specific table — this asset only references the datatable + {parsed.datatable}. +
+ {:else} + + {#key refreshKey} + + {/key} + {/if} +
diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index ad3df85e97..6d45ed3697 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -563,6 +563,9 @@ // the listing immediately — the new (preview or script) job appears in // the history popover without waiting on its 3 s poll tick. let runsRefreshKey = $state(0) + // The most recently dispatched job id — surfaces to AssetRunsPanel so + // the new run auto-selects without an extra click. + let runsPendingJobId = $state(undefined) // Producers (write/rw edges) for the currently-selected asset, derived // from `graphWithDraft.edges`. Threaded into the details pane so the @@ -815,7 +818,10 @@ requestBody: {} }) } - if (jobId) runsRefreshKey++ + if (jobId) { + runsPendingJobId = jobId + runsRefreshKey++ + } return jobId }} /> @@ -826,6 +832,7 @@ selection={activeDraft ? undefined : selection} selectionProducers={activeDraft ? [] : selectionProducers} {runsRefreshKey} + {runsPendingJobId} draftScript={activeDraft?.script} workspace={$workspaceStore} onAnnotationsChange={(scriptPath, annotations) => {