fix: judge dbt relations by provenance, keep dynamic_skip on reopen, refresh saved scripts' titles

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-09-30 12:18:31 +02:00
co-authored by Claude Opus 5.5
parent 43bf7d2d81
commit 21690a2ed8
7 changed files with 64 additions and 40 deletions
@@ -469,14 +469,6 @@
}
})
}
// Set of `script_path` values for runnables that are still drafts (no
// DB row yet). Trigger nodes use this to swap "Click to create" for
// "Click to create (after draft save)" so the user knows the create
// button is blocked until the script is deployed.
const unsavedRunnablePaths = new Set<string>()
for (const r of g.runnables) {
if (r.unsaved) unsavedRunnablePaths.add(r.path)
}
// Producer → its `// data_test` checks, keyed by runnable id, so the
// write-edge to the materialized asset can carry the test badge: the
// edge *is* the transformation, and the tests assert on what it produces.
@@ -842,9 +834,6 @@
missing: info.missing,
draft: info.draft,
runnable_path: info.runnable_path,
runnable_unsaved: info.runnable_path
? unsavedRunnablePaths.has(info.runnable_path)
: false,
// data_upload nodes go green once a file is staged for their
// target script (see readyDataUploadPaths / page dataUploadArgs).
ready: info.runnable_path
@@ -21,8 +21,8 @@ and the controls hung around the card.
selected?: boolean
/** Not deployed yet: dashed edge. */
draft?: boolean
/** Feedback states, which recolor the whole card. */
/** `error` is a misconfigured node, styled like a failed flow step. */
/** Feedback states, which recolor the whole card. `error` is a misconfigured
* node, styled like a failed flow step. */
tone?: 'danger' | 'error' | 'success' | 'running'
/** Makes the whole card a button. */
onclick?: (e: MouseEvent) => void
@@ -24,23 +24,13 @@
type Presentation = {
icon: ComponentType
label: string
// Tailwind class fragments for bg + border + text accent.
bg: string
border: string
borderUnsaved: string
iconText: string
}
// One muted treatment for every source kind — colors are meaningful, not
// One muted icon color for every source kind — colors are meaningful, not
// decorative (brand guidelines), so the kind is carried by the icon and
// label while bg/border stay on the surface/gray scale like the flow
// editor's nodes. Red stays reserved for the missing-trigger state.
const MUTED = {
bg: 'bg-surface-secondary',
border: 'outline-gray-400 dark:outline-gray-600',
borderUnsaved: 'outline-dashed outline-gray-400 dark:outline-gray-500',
iconText: 'text-secondary'
}
// label. Red stays reserved for the missing-trigger state.
const MUTED = { iconText: 'text-secondary' }
export const TRIGGER_NODE_STYLE: Record<TriggerNodeKind, Presentation> = {
schedule: { icon: Clock, label: 'Schedule', ...MUTED },
@@ -83,11 +73,6 @@
// to the draft instead of a trigger row.
draft?: boolean
runnable_path?: string
// True iff the target script is still a draft (no DB row yet).
// Drives the "(after draft save)" hint on the missing
// placeholder, since the page-level handler refuses to open the
// create drawer until the script is deployed.
runnable_unsaved?: boolean
// Page-supplied dispatcher that opens the matching native
// trigger drawer with `script_path` pre-filled. When absent
// (e.g. schedule, or a kind without an editor) the placeholder is
@@ -1274,4 +1274,29 @@ describe('pipelineNodeErrors', () => {
])
expect([...errors.scripts.keys()]).toEqual(['f/x/cons'])
})
it('treats dbt models as used and written by their (hidden) project, but not dbt sources', () => {
const dbt = (path: string, resource_type: string) => ({
kind: 'dbt' as const,
path,
dbt: { unique_id: `x.${path}`, resource_type } as any
})
const errors = pipelineNodeErrors(
baseGraph({
assets: [dbt('mart', 'model'), dbt('raw', 'source')],
runnables: [{ path: 'f/x/s', usage_kind: 'script' }]
}),
new Map([
[
'f/x/s',
[
{ kind: 'dbt', path: 'mart' },
{ kind: 'dbt', path: 'raw' }
]
]
])
)
expect([...errors.assets.keys()]).toEqual(['dbt:raw'])
expect([...errors.scripts.keys()]).toEqual(['f/x/s'])
})
})
@@ -331,7 +331,14 @@ export function pipelineNodeErrors(
.filter((e) => e.access_type === 'w' || e.access_type === 'rw')
.map((e) => `${e.asset_kind}:${e.asset_path}`)
)
for (const e of graph.dbt_edges ?? []) written.add(`dbt:${e.from_asset_path}`)
// The dbt project's own edges are hidden from the pipeline graph (see
// `hideDbtRunnables`), so its relations are judged by their provenance: the
// project uses every one, and materializes all but its sources.
for (const a of graph.assets) {
if (!a.dbt) continue
referenced.add(`${a.kind}:${a.path}`)
if (a.dbt.resource_type !== 'source') written.add(`${a.kind}:${a.path}`)
}
const errors = new Map<string, string>()
const scriptErrors = new Map<string, string>()
for (const a of graph.assets) {
@@ -379,6 +379,7 @@
no_flow_overlap = s?.no_flow_overlap ?? false
wsErrorHandlerMuted = s?.ws_error_handler_muted ?? false
retry = s?.retry ?? undefined
dynamicSkipPath = s?.dynamic_skip ?? undefined
await setScheduleHandler(s)
permissionedAs = undefined
@@ -296,20 +296,21 @@
}
}
// Only-add cache of (script_path → body content) populated lazily by
// `bodyFetchEffect` for every script in the current folder. We never
// remove entries: stale keys (renamed-away, deleted) are simply ignored
// at read time because the derived maps below only iterate paths that
// appear in the current `g.runnables`. That self-cleaning property is
// the whole reason for the refactor — no rename/delete cleanup needed.
// Cache of (script_path → body content) populated lazily by
// `bodyFetchEffect` for every script in the current folder. Stale keys
// (renamed-away, deleted) are simply ignored at read time because the
// derived maps below only iterate paths that appear in the current
// `g.runnables`, so no rename/delete cleanup is needed. A stale *value*
// under a live key is different: a save through this page evicts its path
// (`forgetScriptCache`) so the sweep re-reads it.
let bodiesByPath = $state<Map<string, string>>(new Map())
// Deployed summary + language from the same fetch, for node titles and icons.
// Same only-add cache.
// Same lifecycle as `bodiesByPath`.
let scriptMetaByPath = $state<Map<string, { summary?: string; language: ScriptLang }>>(
new Map()
)
// Sibling cache: the parsed asset usages from `inferAssets` (wasm), one
// pass per body. Same only-add semantics as `bodiesByPath`.
// pass per body. Same lifecycle as `bodiesByPath`.
let inferredAssetsByPath = $state<Map<string, AssetWithAltAccessType[]>>(new Map())
// Bumped on folder change so an in-flight prefetch sweep stops before
// writing into the new folder's state.
@@ -852,6 +853,20 @@
saveErrors = next
}
function forgetScriptCache(...paths: string[]) {
const bodies = new Map(bodiesByPath)
const meta = new Map(scriptMetaByPath)
const inferred = new Map(inferredAssetsByPath)
for (const p of paths) {
bodies.delete(p)
meta.delete(p)
inferred.delete(p)
}
bodiesByPath = bodies
scriptMetaByPath = meta
inferredAssetsByPath = inferred
}
// "This path is gone" cleanup. The three big inferred-* / annotated-*
// maps used to live here too, but they're now derived from
// `bodiesByPath` × `g.runnables` — entries for missing paths drop out
@@ -2860,6 +2875,7 @@
}}
onPersistedSaved={async (savedPath) => {
const predicted = predictCascadeFacts([savedPath])
forgetScriptCache(savedPath)
await graphRes.refetch()
reportDeployDrift(predicted)
}}
@@ -2867,6 +2883,7 @@
if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) {
pe.selection = { ...pe.selection, path: newPath }
}
forgetScriptCache(oldPath, newPath)
await graphRes.refetch()
}}
onScriptRemoved={async (removedPath) => {