mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-07 08:02:40 +00:00
feat(pipelines): require data uploads before running a pipeline (#9953)
* feat(pipelines): require data uploads before running a pipeline Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(pipelines): require every S3Object filled for data-upload readiness Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * feat(pipelines): Run pipeline defaults schedule-triggered scripts to their schedule args Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * docs: correct why only schedules default their args in Run pipeline Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(pipelines): close cascade double-start race and gate data-upload readiness on full-schema validity Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix: expose ScriptEditor validity via callback, not banned bindable-with-default Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
4b1f2207b8
commit
a1c5b7aa3e
@@ -164,6 +164,11 @@
|
||||
// succeeded.
|
||||
requireValidAssets?: boolean
|
||||
args: Record<string, any>
|
||||
// Emitted with the test-form's full-schema validity whenever it changes, so
|
||||
// a host (the pipeline editor) can gate a data-upload entry's readiness on
|
||||
// whether every required field is filled, not just the S3 file. A callback
|
||||
// rather than a bindable so we don't hit the `$bindable(default)` ban.
|
||||
onIsValidChange?: (isValid: boolean) => void
|
||||
// Custom timeout (in seconds) from the script settings. Forwarded to the
|
||||
// preview run so "Test" honors the same timeout a deployed run would,
|
||||
// instead of silently falling back to the instance default.
|
||||
@@ -235,6 +240,7 @@
|
||||
customUi = undefined,
|
||||
requireValidAssets = false,
|
||||
args = $bindable(),
|
||||
onIsValidChange,
|
||||
timeout = undefined,
|
||||
selectedTab = $bindable('main'),
|
||||
hasPreprocessor = $bindable(false),
|
||||
@@ -678,6 +684,10 @@
|
||||
let jobLoader: JobLoader | undefined = $state(undefined)
|
||||
|
||||
let isValid: boolean = $state(true)
|
||||
// Mirror the test-form validity out to an optional host callback.
|
||||
$effect(() => {
|
||||
onIsValidChange?.(isValid)
|
||||
})
|
||||
let scriptProgress = $state(undefined)
|
||||
|
||||
let logPanel: LogPanel | undefined = $state(undefined)
|
||||
|
||||
@@ -129,6 +129,9 @@
|
||||
// form (with the auto-generated S3 picker) for the given script.
|
||||
// data_upload has no trigger row; it's a UI-first entry point.
|
||||
onOpenDataUpload?: (scriptPath: string) => void
|
||||
// Data-upload entry scripts whose file has been staged in the run form —
|
||||
// their source node renders green (ready) instead of the neutral prompt.
|
||||
readyDataUploadPaths?: ReadonlySet<string>
|
||||
// Script paths the cursor is over in the Activity panel — a thin neutral
|
||||
// ring each. One entry for a single run row; the whole cascade's runs
|
||||
// when hovering a group header. Distinct from `selectedRunPaths`.
|
||||
@@ -186,6 +189,7 @@
|
||||
onDeleteTrigger,
|
||||
onOpenWebhook,
|
||||
onOpenDataUpload,
|
||||
readyDataUploadPaths,
|
||||
hoveredPaths,
|
||||
selectedRunPaths,
|
||||
panToNodeId,
|
||||
@@ -703,6 +707,11 @@
|
||||
runnable_unsaved: info.runnable_path
|
||||
? unsavedRunnablePaths.has(info.runnable_path)
|
||||
: false,
|
||||
// data_upload nodes go green once a file is staged for their
|
||||
// target script (see readyDataUploadPaths / page dataUploadArgs).
|
||||
ready: info.runnable_path
|
||||
? (readyDataUploadPaths?.has(info.runnable_path) ?? false)
|
||||
: false,
|
||||
onCreateMissingTrigger,
|
||||
onEditTrigger,
|
||||
onDeleteTrigger,
|
||||
|
||||
@@ -245,6 +245,14 @@
|
||||
// preview only — the client orchestrates the chain). Unset on the deployed
|
||||
// pane, whose single run already cascades via the backend dispatcher.
|
||||
onRunCascadeByPath?: (path: string, args: Record<string, any>) => Promise<string | undefined>
|
||||
// Persisted run-form args for the open data-upload entry (its staged
|
||||
// S3Object), so re-opening the node restores the picked file.
|
||||
runFormInitialArgs?: Record<string, any>
|
||||
// Emitted when the run form's args (or validity) change (read-only
|
||||
// PipelineScriptView OR the edit-mode ScriptEditor test form), so the page
|
||||
// can persist a data-upload entry's staged input (drives node readiness).
|
||||
// `isValid` is the full-schema validity, not just "file present".
|
||||
onRunFormArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
|
||||
}
|
||||
let {
|
||||
selection,
|
||||
@@ -285,7 +293,9 @@
|
||||
onRequestEdit,
|
||||
canRunByPath = false,
|
||||
onRunByPath,
|
||||
onRunCascadeByPath
|
||||
onRunCascadeByPath,
|
||||
runFormInitialArgs,
|
||||
onRunFormArgsChange
|
||||
}: Props = $props()
|
||||
|
||||
let readOnly = $derived(mode !== 'edit')
|
||||
@@ -463,6 +473,31 @@
|
||||
script = undefined
|
||||
})
|
||||
|
||||
// Data-upload capture (edit mode): the ScriptEditor test form binds `args`, so
|
||||
// mirror it up to the page — that stages a data-upload entry's uploaded /
|
||||
// entered input, driving the node's green "ready" state and seeding the
|
||||
// whole-pipeline run. Reset per script in a pre-effect (before the keyed
|
||||
// ScriptEditor remounts) so switching nodes never leaks one node's input into
|
||||
// the next, seeding from the page's persisted staging so re-opening a node
|
||||
// restores its input. Guarded on the path so a staging round-trip
|
||||
// (emit → page → runFormInitialArgs) doesn't re-seed and loop. The read-only
|
||||
// branch uses PipelineScriptView's own onArgsChange instead.
|
||||
let argsSeedPath: string | undefined = undefined
|
||||
$effect.pre(() => {
|
||||
const p = script?.path
|
||||
if (p === argsSeedPath) return
|
||||
argsSeedPath = p
|
||||
args = runFormInitialArgs ? structuredClone($state.snapshot(runFormInitialArgs)) : {}
|
||||
})
|
||||
// Bound out of ScriptEditor's test-form SchemaForm — full-schema validity of
|
||||
// the edit-mode run form, so the page's readiness check sees whether every
|
||||
// required field (not just the S3 file) is satisfied.
|
||||
let runFormIsValid = $state(true)
|
||||
$effect(() => {
|
||||
if (readOnly || !script) return
|
||||
onRunFormArgsChange?.(script.path, $state.snapshot(args), runFormIsValid)
|
||||
})
|
||||
|
||||
// Persist draft edits back to the parent's drafts Map on transitions
|
||||
// (selection change, pane close), not on every keystroke — a per-key
|
||||
// sync triggered drafts → activeDraft → draftScript → re-clone → emit
|
||||
@@ -1172,6 +1207,8 @@
|
||||
downstreamCount={downstreamSubscribers}
|
||||
{runsRefreshKey}
|
||||
{runsPendingJobId}
|
||||
initialArgs={runFormInitialArgs}
|
||||
onArgsChange={onRunFormArgsChange}
|
||||
onRunCompleted={() => {
|
||||
previewRefreshKey += 1
|
||||
onRunCompleted?.()
|
||||
@@ -1234,7 +1271,8 @@
|
||||
bind:assets={liveBodyAssets}
|
||||
bind:inferredColumnLineage={liveColumnLineage}
|
||||
{onTestStateChange}
|
||||
{args}
|
||||
bind:args
|
||||
onIsValidChange={(v) => (runFormIsValid = v)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -69,6 +69,9 @@
|
||||
canRunByPath = false,
|
||||
onRunByPath,
|
||||
onRunCascadeByPath,
|
||||
runFormInitialArgs,
|
||||
onRunFormArgsChange,
|
||||
readyDataUploadPaths,
|
||||
resolveLocalScript,
|
||||
localScriptsVersion,
|
||||
selectionProducers = [],
|
||||
@@ -162,6 +165,12 @@
|
||||
// Run the open script AND its downstream closure (dev preview only; the
|
||||
// deployed pane leaves this unset — its single run cascades via the backend).
|
||||
onRunCascadeByPath?: (path: string, args: Record<string, any>) => Promise<string | undefined>
|
||||
// Persisted run-form args for the currently-open data-upload entry, and a
|
||||
// callback to persist them as they change — see the page's dataUploadArgs.
|
||||
runFormInitialArgs?: Record<string, any>
|
||||
onRunFormArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
|
||||
// Data-upload entry scripts whose staged upload is ready (green node).
|
||||
readyDataUploadPaths?: Set<string>
|
||||
/** Local-dev (`/pipeline_dev`): resolve a node to its working-tree content
|
||||
* so the details pane skips the (nonexistent) deployed-script fetch. May
|
||||
* be async (to infer the args schema for the run form). */
|
||||
@@ -438,6 +447,7 @@
|
||||
{onDeleteTrigger}
|
||||
{onOpenWebhook}
|
||||
{onOpenDataUpload}
|
||||
{readyDataUploadPaths}
|
||||
onselect={onSelect}
|
||||
{onAddScriptForAsset}
|
||||
{onAddPipelineScript}
|
||||
@@ -488,6 +498,8 @@
|
||||
{canRunByPath}
|
||||
{onRunByPath}
|
||||
{onRunCascadeByPath}
|
||||
{runFormInitialArgs}
|
||||
{onRunFormArgsChange}
|
||||
{resolveLocalScript}
|
||||
{localScriptsVersion}
|
||||
selection={activeDraft ? undefined : editor.selection}
|
||||
|
||||
@@ -33,6 +33,15 @@
|
||||
runsRefreshKey?: any
|
||||
runsPendingJobId?: string | undefined
|
||||
onRunCompleted?: () => void
|
||||
// Seed the run form with previously-staged args (e.g. a data-upload entry
|
||||
// whose S3Object was picked earlier and persisted at the page level). Lets
|
||||
// a re-opened node show the file it already has instead of a blank form.
|
||||
initialArgs?: Record<string, any>
|
||||
// Emitted whenever the run form's args (or validity) change, so the page can
|
||||
// persist a data-upload entry's staged input — that drives the node's
|
||||
// "green / ready" state and seeds the whole-pipeline run. `isValid` is the
|
||||
// full-schema validity (all required fields, not just the S3 file).
|
||||
onArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
|
||||
}
|
||||
|
||||
let {
|
||||
@@ -44,12 +53,25 @@
|
||||
downstreamCount = 0,
|
||||
runsRefreshKey,
|
||||
runsPendingJobId,
|
||||
onRunCompleted
|
||||
onRunCompleted,
|
||||
initialArgs,
|
||||
onArgsChange
|
||||
}: Props = $props()
|
||||
|
||||
let args = $state<Record<string, any>>({})
|
||||
// Seed once from the persisted args (see initialArgs). Cloned so the run
|
||||
// form mutates its own copy, not the page's stored snapshot.
|
||||
// svelte-ignore state_referenced_locally
|
||||
let args = $state<Record<string, any>>(
|
||||
initialArgs ? structuredClone($state.snapshot(initialArgs)) : {}
|
||||
)
|
||||
let isValid = $state(true)
|
||||
let running = $state(false)
|
||||
// Push args + validity back up so the page can persist a staged data-upload
|
||||
// entry. Reading the deep snapshot tracks nested changes (e.g. an S3Object's
|
||||
// `s3` field, or items added to a required array).
|
||||
$effect(() => {
|
||||
onArgsChange?.(script.path, $state.snapshot(args), isValid)
|
||||
})
|
||||
// Offer "Run + downstream" only when a cascade dispatch is wired (dev preview)
|
||||
// and the script actually has subscribers to fan out to.
|
||||
let hasCascade = $derived(!!onRunCascade && downstreamCount > 0)
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
import { Handle, Position } from '@xyflow/svelte'
|
||||
import { NODE } from '$lib/components/graph/util'
|
||||
import { twMerge } from 'tailwind-merge'
|
||||
import { AlertTriangle, EllipsisVertical, Target, Trash2 } from 'lucide-svelte'
|
||||
import { AlertTriangle, CheckCircle2, EllipsisVertical, Target, Trash2 } from 'lucide-svelte'
|
||||
import DropdownV2 from '$lib/components/DropdownV2.svelte'
|
||||
import { stopPropagation, preventDefault } from 'svelte/legacy'
|
||||
import type { Item } from '$lib/utils'
|
||||
@@ -101,6 +101,9 @@
|
||||
// auto-generated S3 picker lets the user upload + run) instead of
|
||||
// rendering a "missing" placeholder.
|
||||
onOpenDataUpload?: (scriptPath: string) => void
|
||||
// True once a file is staged for this data_upload entry — renders the
|
||||
// node green so the user knows the pipeline can run (see WIN-2129).
|
||||
ready?: boolean
|
||||
// Page-supplied dispatcher to open the matching native trigger
|
||||
// drawer in edit mode for an attached (non-missing) trigger.
|
||||
// `triggerPath` is the trigger row's path (e.g. the mqtt_trigger
|
||||
@@ -149,6 +152,10 @@
|
||||
// data_upload routes through its own handler — clicking opens the target
|
||||
// script's run form (with the auto-generated S3 picker).
|
||||
let canOpenDataUpload = $derived(isDataUpload && !!data.runnable_path && !!data.onOpenDataUpload)
|
||||
// A staged upload turns the node green (ready to run); before that it stays
|
||||
// on the neutral surface with the "upload a file" prompt.
|
||||
let dataUploadReady = $derived(isDataUpload && data.ready === true)
|
||||
let DataUploadIcon = $derived(dataUploadReady ? CheckCircle2 : style.icon)
|
||||
// Schedule + the other native kinds all have dedicated editors. Webhook and
|
||||
// data_upload are excluded — they route through their own open handlers.
|
||||
let canCreate = $derived(
|
||||
@@ -306,26 +313,47 @@
|
||||
{:else if canOpenDataUpload}
|
||||
<!-- Data upload: UI-first entry point (no trigger row). Clicking the
|
||||
node opens the target script's run form, where the auto-generated
|
||||
S3 picker lets the user upload a file and run the pipeline. Styled
|
||||
like an attached trigger, never the red "missing" state. -->
|
||||
S3 picker lets the user upload a file and run the pipeline. Goes
|
||||
green once a file is staged (ready), so "Run pipeline" can proceed;
|
||||
until then it stays neutral with an "upload a file" prompt. Never
|
||||
the red "missing" state. -->
|
||||
<button
|
||||
type="button"
|
||||
onclick={handleDataUploadClick}
|
||||
class={twMerge(
|
||||
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1 w-full text-left',
|
||||
style.bg,
|
||||
data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border,
|
||||
dataUploadReady
|
||||
? 'bg-green-50 dark:bg-green-900/30 outline-green-500 dark:outline-green-600'
|
||||
: style.bg,
|
||||
dataUploadReady ? '' : data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border,
|
||||
'hover:brightness-95 dark:hover:brightness-110 transition-[filter]'
|
||||
)}
|
||||
style="width: {NODE.width}px; min-height: {NODE.height}px;"
|
||||
title={`Data upload for ${data.runnable_path ?? ''} — click to open the run form and upload a file`}
|
||||
title={dataUploadReady
|
||||
? `Data upload for ${data.runnable_path ?? ''} — a file is staged; the pipeline is ready to run. Click to change it.`
|
||||
: `Data upload for ${data.runnable_path ?? ''} — click to open the run form and upload a file`}
|
||||
>
|
||||
<Icon size={14} class={`shrink-0 ml-2 mr-2 ${style.iconText}`} />
|
||||
<DataUploadIcon
|
||||
size={14}
|
||||
class={`shrink-0 ml-2 mr-2 ${dataUploadReady ? 'text-green-600 dark:text-green-400' : style.iconText}`}
|
||||
/>
|
||||
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
|
||||
<span class="text-3xs uppercase tracking-wide truncate text-tertiary">
|
||||
{style.label}
|
||||
<span
|
||||
class={twMerge(
|
||||
'text-3xs uppercase tracking-wide truncate',
|
||||
dataUploadReady ? 'text-green-700 dark:text-green-400' : 'text-tertiary'
|
||||
)}
|
||||
>
|
||||
{style.label}{dataUploadReady ? ' · ready' : ''}
|
||||
</span>
|
||||
<span
|
||||
class={twMerge(
|
||||
'text-2xs font-mono truncate',
|
||||
dataUploadReady ? 'text-green-700 dark:text-green-400' : 'text-emphasis'
|
||||
)}
|
||||
>
|
||||
{dataUploadReady ? 'File staged' : 'Upload & run'}
|
||||
</span>
|
||||
<span class="text-2xs font-mono truncate text-emphasis"> Upload & run </span>
|
||||
</div>
|
||||
</button>
|
||||
{:else}
|
||||
|
||||
@@ -94,6 +94,7 @@
|
||||
import {
|
||||
JobService,
|
||||
OpenAPI,
|
||||
ScheduleService,
|
||||
ScriptService,
|
||||
type AssetKind,
|
||||
type Script,
|
||||
@@ -1379,6 +1380,43 @@
|
||||
// Run+downstream affordance against overlapping chains stomping each
|
||||
// other's storage writes.
|
||||
let cascadeRunningRoot = $state<string | undefined>(undefined)
|
||||
|
||||
// Script path → its schedule's configured args, so a manual "Run pipeline"
|
||||
// launches a schedule-triggered script with the same payload a real tick
|
||||
// would (rather than empty args). Schedule is the only trigger that stores a
|
||||
// static args payload; every other trigger (webhook / http / event kinds /
|
||||
// data_upload) receives its input from the live event or request at dispatch
|
||||
// time — a message, an HTTP body, an uploaded file — which doesn't exist
|
||||
// during a manual run, so there's nothing to default and those scripts run
|
||||
// empty (data_upload being fed its staged file instead, see dataUploadArgs).
|
||||
// Resolved on demand right before a run (schedule args aren't in the graph
|
||||
// response); fail-safe — a schedule we can't fetch just contributes nothing.
|
||||
let scheduleArgsByPath: Record<string, Record<string, any>> = {}
|
||||
async function resolveScheduleArgs(scripts: string[]): Promise<void> {
|
||||
scheduleArgsByPath = {}
|
||||
const ws = $workspaceStore
|
||||
if (!ws) return
|
||||
const runSet = new Set(scripts)
|
||||
// First schedule (with args) per in-run script; dedupe schedule fetches.
|
||||
const wanted = new Map<string, string>() // script path → schedule path
|
||||
for (const t of displayGraph.triggers) {
|
||||
if (t.trigger_kind !== 'schedule' || t.runnable_kind !== 'script') continue
|
||||
if (!runSet.has(t.runnable_path) || wanted.has(t.runnable_path)) continue
|
||||
if ((t as any).path) wanted.set(t.runnable_path, (t as any).path)
|
||||
}
|
||||
await Promise.all(
|
||||
[...wanted].map(async ([scriptPath, schedulePath]) => {
|
||||
try {
|
||||
const sched = await ScheduleService.getSchedule({ workspace: ws, path: schedulePath })
|
||||
if (sched?.args && Object.keys(sched.args).length > 0) {
|
||||
scheduleArgsByPath[scriptPath] = sched.args as Record<string, any>
|
||||
}
|
||||
} catch {
|
||||
// schedule unreadable / deleted — leave its script on empty args
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
// Launch one pipeline script for the dev-run cascade. Always passes
|
||||
// _wmill_skip_asset_dispatch — the orchestrator owns the whole closure,
|
||||
// so the backend dispatcher must not double-fire the deployed part of a
|
||||
@@ -1391,6 +1429,12 @@
|
||||
// drafts (same condition as `displayGraph`). Otherwise — View mode with
|
||||
// drafts hidden — a bounded run must execute the *deployed* scripts the
|
||||
// user is looking at, not preview jobs from hidden local drafts.
|
||||
// Default a script's run args to its schedule's stored payload (the only
|
||||
// trigger with static args — see resolveScheduleArgs), then let a
|
||||
// data-upload entry's staged file override. runWholePipeline already
|
||||
// refused to start unless every data-upload entry is staged, so this is
|
||||
// always the intended input.
|
||||
const staged = { ...(scheduleArgsByPath[path] ?? {}), ...(dataUploadArgs[path]?.args ?? {}) }
|
||||
const draft = mode === 'edit' || includeDrafts ? pe.drafts.get(path) : undefined
|
||||
if (draft) {
|
||||
if (!draft.script.content || !draft.script.language) {
|
||||
@@ -1402,14 +1446,14 @@
|
||||
content: draft.script.content,
|
||||
language: draft.script.language,
|
||||
path,
|
||||
args: { _wmill_skip_asset_dispatch: true }
|
||||
args: { ...staged, _wmill_skip_asset_dispatch: true }
|
||||
}
|
||||
})
|
||||
}
|
||||
return await JobService.runScriptByPath({
|
||||
workspace: $workspaceStore,
|
||||
path,
|
||||
requestBody: { _wmill_skip_asset_dispatch: true }
|
||||
requestBody: { ...staged, _wmill_skip_asset_dispatch: true }
|
||||
})
|
||||
}
|
||||
// Poll a launched cascade job to a terminal state. Modest fixed cadence —
|
||||
@@ -1456,9 +1500,13 @@
|
||||
true
|
||||
)
|
||||
}
|
||||
// Claim the running-guard BEFORE the first await so a rapid second click
|
||||
// (which reads `cascadeRunningRoot`) can't slip through and double-launch.
|
||||
cascadeRunningRoot = rootPath
|
||||
let rootJobId: string | undefined
|
||||
try {
|
||||
// Seed schedule-triggered members with their configured payload.
|
||||
await resolveScheduleArgs([rootPath, ...closure.nodes])
|
||||
const res = await runCascade({
|
||||
closure,
|
||||
root: rootPath,
|
||||
@@ -1637,9 +1685,13 @@
|
||||
sendUserToast('No runnable scripts in this selection', true)
|
||||
return
|
||||
}
|
||||
// Claim the running-guard BEFORE the first await so a rapid second click
|
||||
// (which reads `cascadeRunningRoot`) can't slip through and double-launch.
|
||||
cascadeRunningRoot = schedule.roots[0] ?? scripts[0]
|
||||
let firstJobId: string | undefined
|
||||
try {
|
||||
// Seed schedule-triggered roots with their configured payload.
|
||||
await resolveScheduleArgs(schedule.nodes)
|
||||
const res = await runSelection({
|
||||
schedule,
|
||||
launch: async (path) => {
|
||||
@@ -1679,6 +1731,71 @@
|
||||
}
|
||||
}
|
||||
|
||||
// Staged run-form input for data-upload entry scripts, keyed by path. Lifted
|
||||
// out of the (transient, per-selection) run form so the uploaded/entered data
|
||||
// persists across selection changes — it drives each entry node's green
|
||||
// "ready" state and seeds the whole-pipeline run with that input. `valid` is
|
||||
// the run form's full-schema validity (all required fields satisfied).
|
||||
let dataUploadArgs = $state<Record<string, { args: Record<string, any>; valid: boolean }>>({})
|
||||
|
||||
// An S3Object-shaped value (the file picker writes `{ s3: '<path>' }`).
|
||||
function isS3Object(v: any): boolean {
|
||||
return !!v && typeof v === 'object' && !Array.isArray(v) && 's3' in v
|
||||
}
|
||||
// Whether a staged value carries actual data. Covers the two shapes a
|
||||
// data-upload entry takes: an S3Object (file picker → `{ s3: '<path>' }`) and
|
||||
// a plain required input (e.g. a JSON array pasted into the run form).
|
||||
function hasMeaningfulValue(v: any): boolean {
|
||||
if (v == null) return false
|
||||
if (typeof v === 'string') return v.length > 0
|
||||
if (Array.isArray(v)) return v.length > 0
|
||||
if (typeof v === 'object')
|
||||
return isS3Object(v) ? typeof v.s3 === 'string' && v.s3.length > 0 : Object.keys(v).length > 0
|
||||
return true // numbers / booleans count as provided
|
||||
}
|
||||
// A data-upload entry is ready once the user actually provided its data, not
|
||||
// just opened the form. Two guards: the form's own full-schema `valid` (every
|
||||
// required field — including non-file ones — is satisfied), AND that any
|
||||
// declared S3Object file field actually carries a file (the picker can leave
|
||||
// an empty `{ s3: '' }` on a non-required file field, which `valid` alone
|
||||
// wouldn't catch).
|
||||
function dataUploadReady(path: string): boolean {
|
||||
const staged = dataUploadArgs[path]
|
||||
if (!staged || !staged.valid) return false
|
||||
const values = Object.values(staged.args)
|
||||
const s3s = values.filter(isS3Object)
|
||||
if (s3s.length > 0) return s3s.every(hasMeaningfulValue)
|
||||
return values.some(hasMeaningfulValue)
|
||||
}
|
||||
// Persist the run form's args + validity, but only for data-upload entries —
|
||||
// other run forms (partitioned producers) run their own way and must not be
|
||||
// mistaken for a staged upload. Idempotent: the run form re-emits on every
|
||||
// keystroke/validation pass, so bail when the value is unchanged — otherwise
|
||||
// each emit would reassign `dataUploadArgs`, giving `readyDataUploadPaths` a
|
||||
// fresh Set identity that re-syncs the canvas and re-fires the form's emit
|
||||
// effect (effect_update_depth_exceeded).
|
||||
function stageRunFormArgs(path: string, args: Record<string, any>, valid: boolean) {
|
||||
if (!dataUploadEntryPaths.has(path)) return
|
||||
const prev = dataUploadArgs[path]
|
||||
if (prev && prev.valid === valid && JSON.stringify(prev.args) === JSON.stringify(args)) return
|
||||
dataUploadArgs = { ...dataUploadArgs, [path]: { args, valid } }
|
||||
}
|
||||
// Pipeline scripts that are data-upload entry points (a `data_upload` trigger
|
||||
// in the displayed graph). They can't auto-run — they need an uploaded file
|
||||
// before the pipeline can go (see runWholePipeline's gate + the node's green
|
||||
// state).
|
||||
let dataUploadEntryPaths = $derived(
|
||||
new Set(
|
||||
displayGraph.triggers
|
||||
.filter((t) => t.trigger_kind === 'data_upload' && t.runnable_kind === 'script')
|
||||
.map((t) => t.runnable_path)
|
||||
)
|
||||
)
|
||||
// Of those, the ones with a staged file — drives the green node treatment.
|
||||
let readyDataUploadPaths = $derived(
|
||||
new Set([...dataUploadEntryPaths].filter((p) => dataUploadReady(p)))
|
||||
)
|
||||
|
||||
// Every pipeline-member script, for the always-visible header "Run pipeline"
|
||||
// control. Per-node runs are hover/select-gated on the canvas; this
|
||||
// pipeline-level affordance runs the whole graph without hunting for a root
|
||||
@@ -1695,6 +1812,21 @@
|
||||
// downstream from every source at once. Reuses the same per-hop launch/poll
|
||||
// and one-cascade-at-a-time guard as the node-level chain runs.
|
||||
async function runWholePipeline() {
|
||||
// Data-upload entries can't auto-run with empty args (they'd run against a
|
||||
// missing S3Object). Require every one to be staged (green) first, and
|
||||
// point the user at the first unready node instead of launching a doomed
|
||||
// run.
|
||||
const unready = allPipelineScripts.filter(
|
||||
(p) => dataUploadEntryPaths.has(p) && !dataUploadReady(p)
|
||||
)
|
||||
if (unready.length > 0) {
|
||||
sendUserToast(
|
||||
`Upload data to ${unready.length} data-upload node${unready.length === 1 ? '' : 's'} first — they must be green before the pipeline can run`,
|
||||
true
|
||||
)
|
||||
openDataUploadRun(unready[0])
|
||||
return
|
||||
}
|
||||
await runBoundedCascade(allPipelineScripts)
|
||||
}
|
||||
|
||||
@@ -2328,6 +2460,9 @@
|
||||
onDeleteTrigger={mode === 'edit' ? deleteAttachedTrigger : undefined}
|
||||
onOpenWebhook={openWebhookDrawer}
|
||||
onOpenDataUpload={openDataUploadRun}
|
||||
{readyDataUploadPaths}
|
||||
runFormInitialArgs={openScriptPath ? dataUploadArgs[openScriptPath]?.args : undefined}
|
||||
onRunFormArgsChange={stageRunFormArgs}
|
||||
onSelect={handleCanvasSelect}
|
||||
onAddScriptForAsset={mode === 'edit' ? handleAddScriptForAsset : undefined}
|
||||
onAddPipelineScript={mode === 'edit' ? handleAddPipelineScript : undefined}
|
||||
|
||||
Reference in New Issue
Block a user