From 21690a2ed8d872f400af13b855beecb7dbe75238 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 30 Sep 2026 12:18:31 +0200 Subject: [PATCH] fix: judge dbt relations by provenance, keep dynamic_skip on reopen, refresh saved scripts' titles Co-Authored-By: Claude Opus 5.5 (1M context) --- .../assets/AssetGraph/AssetGraphCanvas.svelte | 11 ------- .../assets/AssetGraph/PipelineNodeCard.svelte | 4 +-- .../assets/AssetGraph/TriggerNode.svelte | 21 ++---------- .../assets/AssetGraph/resolveGraph.test.ts | 25 ++++++++++++++ .../assets/AssetGraph/resolveGraph.ts | 9 ++++- .../schedules/ScheduleEditorInner.svelte | 1 + .../(logged)/pipeline/[folder]/+page.svelte | 33 ++++++++++++++----- 7 files changed, 64 insertions(+), 40 deletions(-) diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 6bdfc0ffab..cb77d16c69 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -469,14 +469,6 @@ } }) } - // Set of `script_path` values for runnables that are still drafts (no - // DB row yet). Trigger nodes use this to swap "Click to create" for - // "Click to create (after draft save)" so the user knows the create - // button is blocked until the script is deployed. - const unsavedRunnablePaths = new Set() - for (const r of g.runnables) { - if (r.unsaved) unsavedRunnablePaths.add(r.path) - } // Producer → its `// data_test` checks, keyed by runnable id, so the // write-edge to the materialized asset can carry the test badge: the // edge *is* the transformation, and the tests assert on what it produces. @@ -842,9 +834,6 @@ missing: info.missing, draft: info.draft, runnable_path: info.runnable_path, - runnable_unsaved: info.runnable_path - ? unsavedRunnablePaths.has(info.runnable_path) - : false, // data_upload nodes go green once a file is staged for their // target script (see readyDataUploadPaths / page dataUploadArgs). ready: info.runnable_path diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineNodeCard.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineNodeCard.svelte index ab8bc65b8a..704ac0e73e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineNodeCard.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineNodeCard.svelte @@ -21,8 +21,8 @@ and the controls hung around the card. selected?: boolean /** Not deployed yet: dashed edge. */ draft?: boolean - /** Feedback states, which recolor the whole card. */ - /** `error` is a misconfigured node, styled like a failed flow step. */ + /** Feedback states, which recolor the whole card. `error` is a misconfigured + * node, styled like a failed flow step. */ tone?: 'danger' | 'error' | 'success' | 'running' /** Makes the whole card a button. */ onclick?: (e: MouseEvent) => void diff --git a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte index bb17f04b83..6651db9241 100644 --- a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte @@ -24,23 +24,13 @@ type Presentation = { icon: ComponentType label: string - // Tailwind class fragments for bg + border + text accent. - bg: string - border: string - borderUnsaved: string iconText: string } - // One muted treatment for every source kind — colors are meaningful, not + // One muted icon color for every source kind — colors are meaningful, not // decorative (brand guidelines), so the kind is carried by the icon and - // label while bg/border stay on the surface/gray scale like the flow - // editor's nodes. Red stays reserved for the missing-trigger state. - const MUTED = { - bg: 'bg-surface-secondary', - border: 'outline-gray-400 dark:outline-gray-600', - borderUnsaved: 'outline-dashed outline-gray-400 dark:outline-gray-500', - iconText: 'text-secondary' - } + // label. Red stays reserved for the missing-trigger state. + const MUTED = { iconText: 'text-secondary' } export const TRIGGER_NODE_STYLE: Record = { schedule: { icon: Clock, label: 'Schedule', ...MUTED }, @@ -83,11 +73,6 @@ // to the draft instead of a trigger row. draft?: boolean runnable_path?: string - // True iff the target script is still a draft (no DB row yet). - // Drives the "(after draft save)" hint on the missing - // placeholder, since the page-level handler refuses to open the - // create drawer until the script is deployed. - runnable_unsaved?: boolean // Page-supplied dispatcher that opens the matching native // trigger drawer with `script_path` pre-filled. When absent // (e.g. schedule, or a kind without an editor) the placeholder is diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index 846cbdddbb..3b6d66c166 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -1274,4 +1274,29 @@ describe('pipelineNodeErrors', () => { ]) expect([...errors.scripts.keys()]).toEqual(['f/x/cons']) }) + + it('treats dbt models as used and written by their (hidden) project, but not dbt sources', () => { + const dbt = (path: string, resource_type: string) => ({ + kind: 'dbt' as const, + path, + dbt: { unique_id: `x.${path}`, resource_type } as any + }) + const errors = pipelineNodeErrors( + baseGraph({ + assets: [dbt('mart', 'model'), dbt('raw', 'source')], + runnables: [{ path: 'f/x/s', usage_kind: 'script' }] + }), + new Map([ + [ + 'f/x/s', + [ + { kind: 'dbt', path: 'mart' }, + { kind: 'dbt', path: 'raw' } + ] + ] + ]) + ) + expect([...errors.assets.keys()]).toEqual(['dbt:raw']) + expect([...errors.scripts.keys()]).toEqual(['f/x/s']) + }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index d55c18a9ca..aa98caa923 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -331,7 +331,14 @@ export function pipelineNodeErrors( .filter((e) => e.access_type === 'w' || e.access_type === 'rw') .map((e) => `${e.asset_kind}:${e.asset_path}`) ) - for (const e of graph.dbt_edges ?? []) written.add(`dbt:${e.from_asset_path}`) + // The dbt project's own edges are hidden from the pipeline graph (see + // `hideDbtRunnables`), so its relations are judged by their provenance: the + // project uses every one, and materializes all but its sources. + for (const a of graph.assets) { + if (!a.dbt) continue + referenced.add(`${a.kind}:${a.path}`) + if (a.dbt.resource_type !== 'source') written.add(`${a.kind}:${a.path}`) + } const errors = new Map() const scriptErrors = new Map() for (const a of graph.assets) { diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 1317da425b..8f87e9d0d3 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -379,6 +379,7 @@ no_flow_overlap = s?.no_flow_overlap ?? false wsErrorHandlerMuted = s?.ws_error_handler_muted ?? false retry = s?.retry ?? undefined + dynamicSkipPath = s?.dynamic_skip ?? undefined await setScheduleHandler(s) permissionedAs = undefined diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 8bd35f5d46..28bed661e4 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -296,20 +296,21 @@ } } - // Only-add cache of (script_path → body content) populated lazily by - // `bodyFetchEffect` for every script in the current folder. We never - // remove entries: stale keys (renamed-away, deleted) are simply ignored - // at read time because the derived maps below only iterate paths that - // appear in the current `g.runnables`. That self-cleaning property is - // the whole reason for the refactor — no rename/delete cleanup needed. + // Cache of (script_path → body content) populated lazily by + // `bodyFetchEffect` for every script in the current folder. Stale keys + // (renamed-away, deleted) are simply ignored at read time because the + // derived maps below only iterate paths that appear in the current + // `g.runnables`, so no rename/delete cleanup is needed. A stale *value* + // under a live key is different: a save through this page evicts its path + // (`forgetScriptCache`) so the sweep re-reads it. let bodiesByPath = $state>(new Map()) // Deployed summary + language from the same fetch, for node titles and icons. - // Same only-add cache. + // Same lifecycle as `bodiesByPath`. let scriptMetaByPath = $state>( new Map() ) // Sibling cache: the parsed asset usages from `inferAssets` (wasm), one - // pass per body. Same only-add semantics as `bodiesByPath`. + // pass per body. Same lifecycle as `bodiesByPath`. let inferredAssetsByPath = $state>(new Map()) // Bumped on folder change so an in-flight prefetch sweep stops before // writing into the new folder's state. @@ -852,6 +853,20 @@ saveErrors = next } + function forgetScriptCache(...paths: string[]) { + const bodies = new Map(bodiesByPath) + const meta = new Map(scriptMetaByPath) + const inferred = new Map(inferredAssetsByPath) + for (const p of paths) { + bodies.delete(p) + meta.delete(p) + inferred.delete(p) + } + bodiesByPath = bodies + scriptMetaByPath = meta + inferredAssetsByPath = inferred + } + // "This path is gone" cleanup. The three big inferred-* / annotated-* // maps used to live here too, but they're now derived from // `bodiesByPath` × `g.runnables` — entries for missing paths drop out @@ -2860,6 +2875,7 @@ }} onPersistedSaved={async (savedPath) => { const predicted = predictCascadeFacts([savedPath]) + forgetScriptCache(savedPath) await graphRes.refetch() reportDeployDrift(predicted) }} @@ -2867,6 +2883,7 @@ if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { pe.selection = { ...pe.selection, path: newPath } } + forgetScriptCache(oldPath, newPath) await graphRes.refetch() }} onScriptRemoved={async (removedPath) => {