From 2493eaf031f30072637a297398674e761f039005 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 29 Jun 2026 18:19:31 +0200 Subject: [PATCH 01/56] feat(home): redesign create-new popover and home header (#9827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(home): redesign create-new popover and home header Replace the home page "Home" title with a hover-driven "New" popover (CreateActionsMenu) listing Script / Flow / Workflow-as-Code / Apps with a description pane. Workflow-as-Code offers a Python / TypeScript choice; other entries are created by clicking the list row. Move CLI/MCP to the header far right, add a Hub link button, and drop the Workspace/Hub tab switcher so the home page shows only the workspace list. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): keep Home title, move New to the right, swap popover panes Reintroduce the "Home" header title on the left and place the New popover on the right alongside the Hub and CLI/MCP buttons (top-aligned, with extra gap before New). Swap the popover panes so the description is on the left and the option list on the right; the menu opens leftward again. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): make Hub, CLI/MCP and New header buttons uniform md size Set all three header buttons to unifiedSize="md" (New keeps the accent variant to stand out) and re-center the right group now that heights match. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): remove divider between popover panes Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): badge Workflow-as-Code as Advanced and low-code App as Legacy Add inline pills (Advanced / Legacy) next to the option label and in the description header, and widen the option list so the longest label plus badge fits without truncating. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): reorder create options and share App icon Order is now Script, Flow, App (full-code), Workflow-as-Code, App (low-code). Full-code App reuses the low-code App dashboard icon (distinguished by accent). Broaden Option.icon to also accept the BarsStaggered (Flow) component. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): add import / pipeline secondary actions to popover detail panels Surface the previous create-menu extras in the matching detail panel: Flow → Import flow + Pipeline (alpha); Workflow-as-Code → Import Workflow-as-Code; App (full/low-code) → Import full/low-code app. A shared YAML/JSON import drawer parses the pasted source into the relevant store (or sessionStorage for the full-reload apps_raw route) and navigates. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): add Data pipelines editor option (alpha) to create popover Add a first-class "Data pipelines editor" entry right after Workflow-as-Code (indigo accent, Workflow icon, emerald Alpha badge) routing to /pipeline, and drop the now-redundant "Pipeline (alpha)" secondary action from the Flow panel. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): emphasize import buttons and group the badged options Render the detail-panel import actions as default (bordered) buttons with an import icon instead of subtle text, and add a separator in the option list between the three plain options and the three badged (Advanced/Alpha/Legacy) ones. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): increase the y gap around the option-group separator Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): rename Data pipelines editor option to Data pipelines Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): click-open create popover with import submenu and toggleable docs Co-Authored-By: Claude Opus 4.8 (1M context) * feat(home): keyboard-navigable create popover via melt dropdown with looping Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Guilhem Lemouel --- .../components/home/CreateActionsMenu.svelte | 547 ++++++++++++++++++ .../src/lib/components/home/ItemsList.svelte | 3 +- .../src/routes/(root)/(logged)/+page.svelte | 77 +-- 3 files changed, 579 insertions(+), 48 deletions(-) create mode 100644 frontend/src/lib/components/home/CreateActionsMenu.svelte diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte new file mode 100644 index 0000000000..146e390419 --- /dev/null +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -0,0 +1,547 @@ + + +
+ + + {#if $open && active} +
+ {#if showDoc} + +
+
+
+ +
+
+
+

{active.label}

+ {#if active.badge} + + {active.badge.label} + + {/if} +
+

{active.tagline}

+
+
+ +

{active.description}

+ +
    + {#each active.bullets as bullet (bullet)} +
  • + + {bullet} +
    • + {/each} +
+ + +
+ {/if} + + +
+ {#snippet rowBody(option: Option, ac: (typeof accentClasses)[string])} +
+ +
+ + {option.label} + + {#if option.badge} + + {option.badge.label} + + {/if} + {/snippet} + {#each allOptions as option (option.key)} + {@const ac = accentClasses[option.accent]} + {@const rowClass = + 'w-full flex flex-row items-center gap-2.5 rounded-md px-2 py-1.5 text-left cursor-pointer transition-colors focus:outline-none data-[highlighted]:bg-surface-hover hover:bg-surface-hover'} + {#if option.variants} + + {#if $wacSubOpen} +
+ {#each option.variants ?? [] as variant (variant.label)} + {@const VariantIcon = variant.icon} + + {/each} +
+ {/if} + {:else} + + {/if} + {/each} + + +
+ + {#if $importSubOpen} +
+ {#each importActions as action (action.label)} + + {/each} +
+ {/if} + + {#if !showDoc} + + {/if} +
+
+ {/if} +
+ + + + importDrawer?.closeDrawer?.()}> + + + + {#snippet content()} +
+ {#key importType} + {#await import('$lib/components/SimpleEditor.svelte')} + + {:then Module} + + {/await} + {/key} +
+ {/snippet} + + {#snippet actions()} + + {/snippet} + + diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 256b2277f5..b6ae577d4b 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -506,7 +506,8 @@ if (menuItem) { if (e.key === 'ArrowUp' || e.key === 'ArrowDown') { const menu = menuItem.closest('[role="menu"]') - if (menu) { + // menus marked data-arrow-loop keep melt's cyclic wrap instead of exiting + if (menu && !menu.hasAttribute('data-arrow-loop')) { const items = Array.from(menu.querySelectorAll('[role="menuitem"]')) const idx = items.indexOf(menuItem) const isFirst = idx === 0 diff --git a/frontend/src/routes/(root)/(logged)/+page.svelte b/frontend/src/routes/(root)/(logged)/+page.svelte index b070bd5fef..e326bc87f7 100644 --- a/frontend/src/routes/(root)/(logged)/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/+page.svelte @@ -1,13 +1,11 @@ + + + +
+ + {#if boundBar}{@render boundBar()}{/if} + {#if mode === 'edit'} + + {/if} + {#if prefetchingAssets} +
+ + Parsing assets… +
+ {/if} + {#if onTogglePanelHidden && (mode !== 'edit' || editor.selection != undefined || editor.activeDraftPath != undefined)} +
+
+ {/if} +
+ {#if detailsPaneOpen && workspace} + + {#if idleView && idlePane} + {@render idlePane()} + {:else} + onStartBoundedRunForOpen?.(editor.openScriptPath!) + : undefined} + {onRunCompleted} + {onTestStateChange} + {requestRemoveSignal} + {requestRunSignal} + {requestRunCascadeSignal} + {focusUploadSignal} + draftScript={activeDraft?.script} + {pathPrefix} + {onDraftPathChange} + {workspace} + onAnnotationsChange={editor.handleAnnotationsChange} + onAssetsChange={editor.handleAssetsChange} + onContentChange={editor.handleContentChange} + onDraftPersist={editor.handleDraftPersist} + onclose={onClose} + onHide={onTogglePanelHidden} + {onDiscard} + {onDraftSaved} + {onPersistedSaved} + {onScriptRenamed} + {onScriptRemoved} + /> + {/if} + + {/if} + diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts new file mode 100644 index 0000000000..a1a82870c8 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts @@ -0,0 +1,143 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { JobService, ScriptService } from '$lib/gen' +import { createPipelineAiHelpers, type PipelineDraft } from './pipelineAiHelpers' +import type { AssetGraphResponse } from './types' + +// Build a helper handle over an in-memory drafts Map, mirroring how the editor +// wires it. `getFolder` returns the bare folder name (as the route/session do). +function makeHandle( + initial: Array<[string, PipelineDraft]> = [], + runnables: Array<{ path: string }> = [] +) { + let drafts = new Map(initial) + let forgotten: string[] = [] + const handle = createPipelineAiHelpers({ + getFolder: () => 'x', + getWorkspace: () => 'w', + getResolvedGraph: () => + ({ assets: [], runnables, edges: [], triggers: [] }) as unknown as AssetGraphResponse, + getDrafts: () => drafts, + setDrafts: (next) => (drafts = next), + newDraftLocalId: () => 'id', + onForgetPath: (p) => forgotten.push(p) + }) + return { handle, drafts: () => drafts, forgotten: () => forgotten } +} + +afterEach(() => vi.restoreAllMocks()) + +const draft = (over: Partial = {}): PipelineDraft => + ({ localId: 'l', script: { content: '' } as any, ...over }) as PipelineDraft + +describe('pipeline AI direct-draft helpers', () => { + it('removeProposedNode discards the unsaved draft at a path', async () => { + const { handle, drafts, forgotten } = makeHandle([ + ['f/x/a', draft()], + ['f/x/b', draft()] + ]) + await handle.removeProposedNode('f/x/a') + expect(drafts().has('f/x/a')).toBe(false) + expect(drafts().has('f/x/b')).toBe(true) + expect(forgotten()).toContain('f/x/a') + }) + + it('removeProposedNode throws when there is no draft to discard', async () => { + const { handle } = makeHandle() + await expect(handle.removeProposedNode('f/x/missing')).rejects.toThrow() + }) + + it('getPipelineContext does not expose any pending/approval state', () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + const ctx = handle.getPipelineContext() + expect(ctx).not.toHaveProperty('pendingProposals') + expect(handle).not.toHaveProperty('acceptAll') + expect(handle).not.toHaveProperty('rejectAll') + }) + + it('testNode on a deployed node never dispatches downstream subscribers', async () => { + // No draft at the path → runs the deployed version, which must carry + // `_wmill_skip_asset_dispatch` so a single-node test can't fire downstream. + const spy = vi.spyOn(JobService, 'runScriptByPath').mockResolvedValue('job-1' as any) + const { handle } = makeHandle() + await handle.testNode('f/x/deployed', { foo: 1 }) + expect(spy).toHaveBeenCalledWith( + expect.objectContaining({ + requestBody: expect.objectContaining({ _wmill_skip_asset_dispatch: true, foo: 1 }) + }) + ) + }) + + it('proposeNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/other/n', language: 'duckdb' as any, content: '' }) + ).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects content missing the pipeline annotation', async () => { + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ path: 'f/x/new', language: 'duckdb' as any, content: 'SELECT 1' }) + ).rejects.toThrow(/pipeline annotation/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path colliding with an existing draft', async () => { + const { handle } = makeHandle([['f/x/a', draft()]]) + await expect( + handle.proposeNode({ path: 'f/x/a', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + }) + + it('proposeNode rejects a path colliding with an existing deployed node', async () => { + const { handle, drafts } = makeHandle([], [{ path: 'f/x/dep' }]) + await expect( + handle.proposeNode({ path: 'f/x/dep', language: 'duckdb' as any, content: '-- pipeline' }) + ).rejects.toThrow(/already exists/) + expect(drafts().size).toBe(0) + }) + + it('proposeNode rejects a path that is an already-deployed script when the graph has not hydrated', async () => { + // Empty graph (session preview can race open_preview), but a deployed script + // exists at the path — the backend probe must still catch it. + const spy = vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue({} as any) + const { handle, drafts } = makeHandle() + await expect( + handle.proposeNode({ + path: 'f/x/deployed', + language: 'duckdb' as any, + content: '-- pipeline' + }) + ).rejects.toThrow(/already exists/) + expect(spy).toHaveBeenCalled() + expect(drafts().size).toBe(0) + }) + + it('editNode rejects a path outside the open folder', async () => { + const { handle, drafts } = makeHandle() + await expect(handle.editNode('f/other/foo', '-- pipeline')).rejects.toThrow(/open folder/) + expect(drafts().size).toBe(0) + }) + + it('editNode preserves the deployed script hash/metadata and replaces only content', async () => { + const deployed = { + hash: 'abc123', + path: 'f/x/node', + summary: 'My node', + description: 'desc', + tag: 'custom', + language: 'duckdb', + content: '-- pipeline\nSELECT 1' + } + vi.spyOn(ScriptService, 'getScriptByPath').mockResolvedValue(deployed as any) + const { handle, drafts } = makeHandle() + await handle.editNode('f/x/node', '-- pipeline\nSELECT 2') + const d = drafts().get('f/x/node') + expect(d?.script.hash).toBe('abc123') + expect(d?.script.summary).toBe('My node') + expect(d?.script.description).toBe('desc') + expect(d?.script.tag).toBe('custom') + expect(d?.script.content).toBe('-- pipeline\nSELECT 2') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts new file mode 100644 index 0000000000..959992543f --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts @@ -0,0 +1,319 @@ +import { JobService, ScriptService, type AssetKind, type Script, type ScriptLang } from '$lib/gen' +import { emptySchema, sendUserToast } from '$lib/utils' +import { inferAssets } from '$lib/infer' +import { extractWrites, type AssetWithAltAccessType } from '$lib/components/assets/lib' +import { assetUri, autoOutputAsset, type PipelineOutputKind } from './pipelineTemplates' +import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import type { AssetGraphResponse } from './types' +import type { + PipelineAIChatHelpers, + PipelineContext, + PipelineNodeSummary +} from '$lib/components/copilot/chat/pipeline/core' + +// ============================================================================ +// Shared data-pipeline AI helper layer. +// +// Both the full-page editor (/pipeline/[folder]) and the in-session preview +// (PipelineEditorView) drive the AI chat's pipeline tools through this factory, +// so the build/edit logic lives in exactly one place. Each caller injects +// accessors for its own draft Map and graph; this module owns the AI behaviour +// (build/edit/discard/test). AI edits apply directly as unsaved drafts — there +// is no separate approve/reject step. +// ============================================================================ + +/** + * An unsaved pipeline node draft. `localId` is a stable per-draft id preserved + * across renames (the page uses it to dedupe concurrent deploys). AI-built nodes + * and manually-created drafts are the same thing — an unsaved node on the canvas. + */ +export type PipelineDraft = { + localId: string + script: Script + outputAssets?: Array<{ kind: AssetKind; path: string }> +} + +export type PipelineAiHelperDeps = { + getFolder: () => string + getWorkspace: () => string | undefined + /** The draft-overlaid graph (resolveGraph output) the context summary reads. */ + getResolvedGraph: () => AssetGraphResponse + getDrafts: () => Map + setDrafts: (next: Map) => void + /** Stable id for a freshly-created draft (route page tracks deploys by it). */ + newDraftLocalId: () => string + /** Focus/select the node after it is staged (pan + open in the pane). */ + onProposeNode?: (path: string) => void + /** Throw (or switch to edit mode) when the surface can't accept AI edits. */ + ensureEditable?: () => void + /** Surface the draft overlay if it is hidden (the page's "show drafts" view). */ + onShowDrafts?: () => void + /** Forget per-path state when a draft is discarded. */ + onForgetPath?: (path: string) => void + /** Notify the caller a test run started so it can light up its run UI. */ + onRunStarted?: (jobId: string, path: string) => void +} + +export function makePipelineScript( + language: ScriptLang, + scriptPath: string, + content: string, + createdAt: string +): Script { + // Cast through unknown: a local draft only needs path/language/content/schema; + // the many readonly deployment fields on Script don't matter until createScript. + return { + hash: '', + path: scriptPath, + summary: '', + description: '', + content, + schema: emptySchema(), + is_template: false, + extra_perms: {}, + language, + kind: 'script', + created_by: '', + created_at: createdAt, + archived: false, + deleted: false, + starred: false + } as unknown as Script +} + +async function inferOutputAssets( + language: ScriptLang, + content: string +): Promise> { + try { + const inferred = await inferAssets(language, content) + if (inferred?.status === 'error') return [] + return extractWrites((inferred?.assets ?? []) as AssetWithAltAccessType[]) + } catch { + return [] + } +} + +export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIChatHelpers { + // A staged draft is always persisted into the OPEN folder's data_pipeline + // bundle, so a path outside the folder would silently land an unrelated script + // there. Both build and edit must stay scoped to the folder. + function assertInFolder(path: string) { + const folder = deps.getFolder() + if (folder && !path.startsWith(`f/${folder}/`)) { + throw new Error( + `Pipeline nodes must be in the open folder — use a path under 'f/${folder}/' (got '${path}').` + ) + } + } + + // A pipeline node IS its `// pipeline` annotation (it's what makes the deployed + // script a pipeline member). Reject content that lacks it so a staged draft + // isn't a non-member script the model can't see is broken until deploy. + function assertPipelineAnnotation(content: string) { + if (!parsePipelineAnnotations(content).inPipeline) { + throw new Error( + `Pipeline node content must declare the pipeline annotation on its own comment line ` + + `(\`// pipeline\`, or \`-- pipeline\` for SQL / \`# pipeline\` for Python).` + ) + } + } + + function buildContext(): PipelineContext { + const graph = deps.getResolvedGraph() + const drafts = deps.getDrafts() + const nodes: PipelineNodeSummary[] = graph.runnables + .filter((r) => r.usage_kind === 'script') + .map((r) => { + const draft = drafts.get(r.path) + const writes = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'w' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const reads = graph.edges + .filter( + (e) => + e.runnable_kind === 'script' && + e.runnable_path === r.path && + (e.access_type === 'r' || e.access_type === 'rw') + ) + .map((e) => assetUri({ kind: e.asset_kind, path: e.asset_path })) + const triggers = graph.triggers + .filter((t) => t.runnable_kind === 'script' && t.runnable_path === r.path) + .map((t) => + t.trigger_kind === 'asset' + ? assetUri({ kind: t.asset_kind, path: t.asset_path }) + : t.trigger_kind + ) + return { + path: r.path, + language: draft?.script.language, + unsaved: r.unsaved ?? false, + summary: draft?.script.summary || undefined, + writes: [...new Set(writes)], + reads: [...new Set(reads)], + triggers: [...new Set(triggers)] + } + }) + return { + folder: deps.getFolder(), + mode: 'edit', + nodes, + assets: graph.assets.map((a) => assetUri({ kind: a.kind, path: a.path })) + } + } + + const helpers: PipelineAIChatHelpers = { + getPipelineContext: buildContext, + getNodeBody: async (path) => { + const draft = deps.getDrafts().get(path) + if (draft) return { language: draft.script.language, content: draft.script.content } + const workspace = deps.getWorkspace() + if (!workspace) return undefined + try { + const deployed = await ScriptService.getScriptByPath({ workspace, path }) + return { language: deployed.language, content: deployed.content } + } catch { + return undefined + } + }, + proposeNode: async ({ path, language, content, outputKind }) => { + deps.ensureEditable?.() + // build_pipeline_node creates a NEW node in the OPEN folder. Reject a path + // outside the folder (it would silently stage into this folder's bundle) + // and a path that collides with an existing node (the model should use + // edit_pipeline_node instead of shadowing a deployed node as a draft). + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + if (drafts.has(path)) { + throw new Error( + `A draft already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + if (deps.getResolvedGraph().runnables.some((r) => r.path === path)) { + throw new Error( + `A pipeline node already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + // Authoritative new-node check: the resolved graph may not have hydrated yet + // (the session preview can race open_preview), and it only lists pipeline + // runnables — so probe the backend. ANY deployed script at this path means + // "build new" would shadow it on deploy; the model should edit instead. + const workspace = deps.getWorkspace() + if (workspace) { + let deployedExists = false + try { + await ScriptService.getScriptByPath({ workspace, path }) + deployedExists = true + } catch { + // 404 → no deployed script at this path, safe to create a new node. + } + if (deployedExists) { + throw new Error( + `A script already exists at '${path}'. Use edit_pipeline_node to change it instead.` + ) + } + } + const inferred = await inferOutputAssets(language, content) + // Fall back to a seeded output (from the declared output_kind) when the + // body doesn't yet write anything inferable. + const seeded = + inferred[0] ?? + (outputKind + ? autoOutputAsset(outputKind as PipelineOutputKind, deps.getFolder(), language) + : undefined) + const next = new Map(drafts) + next.set(path, { + localId: deps.newDraftLocalId(), + script: makePipelineScript(language, path, content, new Date().toISOString()), + outputAssets: inferred.length > 0 ? inferred : seeded ? [seeded] : undefined + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + return { path } + }, + editNode: async (path, content) => { + deps.ensureEditable?.() + assertInFolder(path) + assertPipelineAnnotation(content) + const drafts = deps.getDrafts() + const existing = drafts.get(path) + // Base the edit on the existing draft's / deployed script object and replace + // ONLY the content — preserving hash, summary, description, tag, schema, and + // settings. Rebuilding a fresh script would wipe that metadata: deploying + // from the pane (auto_parent) would update the script while clearing it, and + // the route "Save all" path (no parent_hash) could hit the path-conflict + // branch on the occupied path. + let baseScript: Script + if (existing) { + baseScript = existing.script + } else { + const workspace = deps.getWorkspace() + if (!workspace) throw new Error('No workspace is selected.') + baseScript = await ScriptService.getScriptByPath({ workspace, path }) + } + const inferred = await inferOutputAssets(baseScript.language, content) + const next = new Map(drafts) + next.set(path, { + localId: existing?.localId ?? deps.newDraftLocalId(), + script: { ...baseScript, content }, + outputAssets: inferred.length > 0 ? inferred : existing?.outputAssets + }) + deps.setDrafts(next) + deps.onShowDrafts?.() + deps.onProposeNode?.(path) + }, + removeProposedNode: async (path) => { + if (!deps.getDrafts().has(path)) { + throw new Error(`No unsaved draft at '${path}' to discard.`) + } + const next = new Map(deps.getDrafts()) + next.delete(path) + deps.setDrafts(next) + deps.onForgetPath?.(path) + }, + testNode: async (path, args) => { + const workspace = deps.getWorkspace() + if (!workspace) return undefined + const draft = deps.getDrafts().get(path) + try { + let jobId: string + if (draft) { + // Un-deployed/edited body: preview-run the draft content so it can be + // tested before deploying. + jobId = await JobService.runScriptPreview({ + workspace, + requestBody: { + path, + content: draft.script.content, + language: draft.script.language, + args: args ?? {} + } + }) + } else { + // test_pipeline_node previews ONE node — never fan out to downstream + // deployed subscribers via the backend asset dispatcher (which would + // run side-effecting deployed scripts the user didn't ask for). + jobId = await JobService.runScriptByPath({ + workspace, + path, + requestBody: { ...(args ?? {}), _wmill_skip_asset_dispatch: true } + }) + } + deps.onRunStarted?.(jobId, path) + return jobId + } catch (e: any) { + sendUserToast(`Run failed: ${e?.body ?? e?.message ?? e}`, true) + return undefined + } + } + } + + return helpers +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts new file mode 100644 index 0000000000..bdecadff91 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.svelte.ts @@ -0,0 +1,184 @@ +import type { AssetKind, Script } from '$lib/gen' +import type { AssetWithAltAccessType } from '$lib/components/assets/lib' +import type { AssetGraphSelection } from './types' +import { + parsePipelineAnnotations, + type ColumnLineage, + type PipelineAnnotations +} from './parsePipelineAnnotations' +import type { PipelineDraft } from './pipelineAiHelpers' + +// ============================================================================ +// Externalized pipeline-editor state — the data-pipeline analogue of the flow +// editor's `flowStore` / `flowStateStore`. It owns the in-flight draft Map, the +// live editor overlays, and the current selection: the substrate the route page +// editor and the in-session preview both render through (via the shared +// ). Persistence, graph resolution, run dispatch, and deploy +// stay with the consumer; this is a plain reactive bag so a consumer can +// read/mutate it without prop plumbing. +// ============================================================================ + +const EMPTY_ANNOTATIONS: PipelineAnnotations = parsePipelineAnnotations('') + +type LiveAnnotations = { scriptPath: string | undefined; annotations: PipelineAnnotations } +type LiveBodyAssets = { + scriptPath: string | undefined + assets: AssetWithAltAccessType[] + columnLineage?: ColumnLineage[] +} +type LiveContent = { scriptPath: string | undefined; content: string } + +export class PipelineEditorState { + /** In-flight drafts keyed by script path (manual + AI-staged). */ + drafts = $state>(new Map()) + /** Draft open in the details pane (mutually exclusive with `selection`). */ + activeDraftPath = $state(undefined) + /** The persisted node/asset selected on the canvas. */ + selection = $state(undefined) + + /** Live-parsed annotations of the open script (refreshed per keystroke). */ + liveAnnotations = $state({ + scriptPath: undefined, + annotations: EMPTY_ANNOTATIONS + }) + /** Live-inferred body read/write assets of the open script. */ + liveBodyAssets = $state({ scriptPath: undefined, assets: [] }) + /** The open draft's live editor buffer. */ + liveContent = $state({ scriptPath: undefined, content: '' }) + + /** Set true once a draft bundle was restored from the DB on load — drives the + * route toolbar's one-shot "Loaded from draft" hint. Written by the editor's + * autosave hydrate when persistence is enabled. */ + loadedFromDbDraft = $state(false) + + /** Folder this state is scoped to. Used by the in-session preview (where one + * instance is reused across editor hide/show) to detect a retarget to a + * different folder and reset, so stale drafts don't bleed across folders. */ + folder = $state(undefined) + + /** True once the DB draft bundle for the current folder has been hydrated + * into this instance. Gated per-instance (not per component mount) so the + * in-session preview hydrates ONCE when its runtime is fresh and then keeps + * the in-memory drafts across editor hide/show — re-reading the DB on every + * remount would race a not-yet-flushed autosave and drop a just-staged draft. + * Reset to false on a folder retarget so the new folder re-hydrates. */ + hydratedFromDb = $state(false) + + /** Clear all in-flight state. Used when the session preview retargets a + * different pipeline folder (a same-folder remount keeps the drafts). */ + reset = () => { + this.drafts = new Map() + this.activeDraftPath = undefined + this.selection = undefined + this.clearLiveOverlays() + this.loadedFromDbDraft = false + // Force a re-hydrate from the DB draft of the newly-targeted folder. + this.hydratedFromDb = false + } + + #nextDraftLocalId = 0 + // Arrow fields so `pe.method` can be passed straight as a callback (the + // details pane takes onDraftPersist / onAnnotationsChange / … by reference). + newDraftLocalId = (): string => { + this.#nextDraftLocalId += 1 + return `pe-${this.#nextDraftLocalId}` + } + + handleAnnotationsChange = (scriptPath: string | undefined, annotations: PipelineAnnotations) => { + this.liveAnnotations = { scriptPath, annotations } + } + handleAssetsChange = ( + scriptPath: string | undefined, + assets: AssetWithAltAccessType[], + columnLineage?: ColumnLineage[] + ) => { + this.liveBodyAssets = { scriptPath, assets, columnLineage } + } + handleContentChange = (scriptPath: string | undefined, content: string) => { + this.liveContent = { scriptPath, content } + } + + clearLiveOverlays = () => { + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + this.liveContent = { scriptPath: undefined, content: '' } + } + + /** Drop per-path editor state when a path goes away. Does NOT touch + * consumer-owned per-path state (e.g. the route page's save errors — the + * route layers that on in its own wrapper). */ + forgetPath = (path: string) => { + if (this.activeDraftPath === path) this.activeDraftPath = undefined + if (this.selection?.kind === 'runnable' && this.selection.path === path) + this.selection = undefined + if (this.liveAnnotations.scriptPath === path) + this.liveAnnotations = { scriptPath: undefined, annotations: EMPTY_ANNOTATIONS } + if (this.liveBodyAssets.scriptPath === path) + this.liveBodyAssets = { scriptPath: undefined, assets: [] } + if (this.liveContent.scriptPath === path) + this.liveContent = { scriptPath: undefined, content: '' } + } + + discardDraft = (path: string) => { + if (!this.drafts.has(path)) return + const next = new Map(this.drafts) + next.delete(path) + this.drafts = next + this.forgetPath(path) + } + + /** Commit body edits + inferred outputs back into the drafts Map on pane + * teardown (deferred a microtask so a same-batch discard doesn't resurrect the + * entry). Verbatim port of the route page's `handleDraftPersist`. */ + handleDraftPersist = ( + p: string, + snapshot: { content: string; writes: { kind: AssetKind; path: string }[]; script?: Script } + ) => { + queueMicrotask(() => { + const d = this.drafts.get(p) + if (!d) { + if (!snapshot.script) return + const next = new Map(this.drafts) + next.set(p, { + localId: this.newDraftLocalId(), + script: snapshot.script, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + return + } + // `?? 0` is load-bearing: an undefined `outputAssets` (a no-output draft) + // vs an empty inferred `writes` both mean "no writes". Without the + // coalesce, `undefined === 0` is false, so this never short-circuits — + // every persist re-writes the drafts Map with an equivalent object, + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop (hangs the tab without an effect-depth throw). + const writesEqual = + (d.outputAssets?.length ?? 0) === snapshot.writes.length && + (d.outputAssets ?? []).every( + (a, i) => a.kind === snapshot.writes[i]?.kind && a.path === snapshot.writes[i]?.path + ) + if (d.script.content === snapshot.content && writesEqual) return + const next = new Map(this.drafts) + next.set(p, { + ...d, + script: { ...d.script, content: snapshot.content }, + outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined + }) + this.drafts = next + }) + } + + /** The draft open in the pane, if any. */ + get activeDraft(): PipelineDraft | undefined { + return this.activeDraftPath ? this.drafts.get(this.activeDraftPath) : undefined + } + + /** Whichever script is open — the active draft, or a selected persisted script. */ + get openScriptPath(): string | undefined { + if (this.activeDraftPath) return this.activeDraftPath + if (this.selection?.kind === 'runnable' && this.selection.runnable_kind === 'script') + return this.selection.path + return undefined + } +} diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts new file mode 100644 index 0000000000..0fae3f836d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineEditorState.test.ts @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest' +import { PipelineEditorState } from './pipelineEditorState.svelte' +import type { PipelineDraft } from './pipelineAiHelpers' +import type { AssetKind } from '$lib/gen' + +// handleDraftPersist defers its commit a microtask (so a same-batch discard can +// win); flush that microtask before asserting. +const flushMicrotasks = () => new Promise((resolve) => queueMicrotask(() => resolve())) + +function draft(content: string, outputAssets?: { kind: AssetKind; path: string }[]): PipelineDraft { + return { + localId: 'pe-1', + script: { path: 'f/x/n', language: 'duckdb', content } as PipelineDraft['script'], + outputAssets + } +} + +describe('PipelineEditorState.handleDraftPersist', () => { + // Regression: a no-output draft has `outputAssets: undefined`; the details pane + // infers an empty `writes: []`. Both mean "no writes", so persisting unchanged + // content+writes must be a no-op. The earlier `undefined === 0` length check made + // it false, so every persist re-wrote the drafts Map with an equivalent object — + // re-triggering the pane's emit → graph re-derive → persist, an infinite + // microtask loop that froze the tab. The Map reference must stay identical. + it('is idempotent for a no-output draft (undefined outputAssets vs empty inferred writes)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 1', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) + + it('re-writes the drafts Map when the content actually changes', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { content: 'SELECT 2', writes: [] }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.script.content).toBe('SELECT 2') + }) + + it('re-writes the drafts Map when the inferred writes actually change', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([['f/x/n', draft('SELECT 1', undefined)]]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).not.toBe(before) + expect(pe.drafts.get('f/x/n')?.outputAssets).toEqual([{ kind: 'resource', path: 'f/x/out' }]) + }) + + it('stays idempotent when outputAssets and inferred writes match (non-empty)', async () => { + const pe = new PipelineEditorState() + pe.drafts = new Map([ + ['f/x/n', draft('SELECT 1', [{ kind: 'resource' as AssetKind, path: 'f/x/out' }])] + ]) + const before = pe.drafts + pe.handleDraftPersist('f/x/n', { + content: 'SELECT 1', + writes: [{ kind: 'resource' as AssetKind, path: 'f/x/out' }] + }) + await flushMicrotasks() + expect(pe.drafts).toBe(before) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index dd4c64b225..378c4e1469 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -66,11 +66,14 @@ describe('resolveGraph', () => { expect(resolveGraph(input({ base }))).toEqual(base) }) - it('draft: adds an unsaved runnable + write edge from the static outputAsset', () => { + it('draft: adds an unsaved runnable + write edge from outputAssets', () => { const drafts = new Map([ [ 'f/x/d', - { script: { content: '' }, outputAsset: { kind: 's3object' as const, path: '/out.json' } } + { + script: { content: '' }, + outputAssets: [{ kind: 's3object' as const, path: '/out.json' }] + } ] ]) const r = resolveGraph(input({ drafts })) @@ -93,22 +96,6 @@ describe('resolveGraph', () => { }) }) - it('draft: outputAssets snapshot wins over the static outputAsset', () => { - const drafts = new Map([ - [ - 'f/x/d', - { - script: { content: '' }, - outputAsset: { kind: 's3object' as const, path: '/old.json' }, - outputAssets: [{ kind: 's3object' as const, path: '/new.json' }] - } - ] - ]) - const r = resolveGraph(input({ drafts })) - expect(r.edges.map((e) => e.asset_path)).toContain('/new.json') - expect(r.edges.map((e) => e.asset_path)).not.toContain('/old.json') - }) - it('active draft: live body writes are authoritative over the snapshot', () => { const drafts = new Map([ [ diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index b7c82822fa..e223379adb 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -15,7 +15,6 @@ import { /** Minimal structural shape of a pipeline draft `resolveGraph` needs. */ export type GraphDraft = { script: { content: string } - outputAsset?: { kind: AssetKind; path: string } outputAssets?: Array<{ kind: AssetKind; path: string }> } @@ -280,20 +279,15 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { unsaved: true } } - // Output asset(s): three-tier resolution. + // Output asset(s): two-tier resolution. // 1. Active draft (the body the user is editing right now): // live body inference is authoritative — renaming a // CREATE TABLE target or writeS3File path retires the // old output node and surfaces the new one as the user // types. - // 2. Inactive draft with a captured `outputAssets` snapshot - // (taken on the last pane transition): use those, so a - // draft the user already edited keeps its renamed outputs - // after they've clicked elsewhere. - // 3. Fallback to the static `outputAsset` seeded at draft - // creation — covers fresh drafts and parser misses (e.g. - // WIN-1943: wmill.writeS3File({s3, storage}) object form - // not yet detected by the TS parser). + // 2. Inactive draft: its captured `outputAssets` (inferred at + // creation/last edit, or the seeded output for a fresh draft + // whose body doesn't yet write anything inferable). const liveForThisDraft = liveBodyAssets.scriptPath === path const writeOuts: Array<{ kind: AssetKind; path: string }> = [] if (liveForThisDraft) { @@ -301,9 +295,6 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { } else if (d.outputAssets) { writeOuts.push(...d.outputAssets) } - if (writeOuts.length === 0 && d.outputAsset) { - writeOuts.push(d.outputAsset) - } // `// materialize ` declares a write output via annotation, not // the SQL body, so the body-inference tiers above miss it. Add it from // the live-parsed annotations so an edited materialize script keeps its diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 9276f500af..274d218e82 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -49,6 +49,7 @@ export interface AssetGraphRunnableNode { // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. + // AI-built nodes are plain drafts too (no separate pending/approval state). unsaved?: boolean } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index fdba95ca7f..343965b709 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -88,6 +88,11 @@ import { type GlobalToolHelpers } from './global/core' import { isGlobalAiEnabled } from './global/gate' +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers +} from './pipeline/core' import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userScopedStorage' import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' @@ -254,6 +259,7 @@ export class AIChatManager { skipResponsesApi = false mode = $state(AIMode.NAVIGATOR) + pipelineAiChatHelpers = $state(undefined) readonly isOpen = $derived(chatState.size > 0) savedSize = $state(0) instructions = $state('') @@ -649,18 +655,11 @@ export class AIChatManager { ) switch (result) { case 'ok': - await this.historyManager.saveChat( - this.displayMessages, - this.messages, - this.contextUsage - ) + await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) sendUserToast('Conversation compacted.') break case 'empty': - sendUserToast( - 'Compaction produced an empty summary — conversation left unchanged.', - true - ) + sendUserToast('Compaction produced an empty summary — conversation left unchanged.', true) break case 'error': sendUserToast('Failed to compact the conversation.', true) @@ -958,28 +957,7 @@ export class AIChatManager { this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] this.helpers = {} } else if (mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(mode), { - previewTools: this.isSessionChat, - skills: this.globalSkills - }) - this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) - this.helpers = { - ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), - testActiveFlow: async (args?: Record) => - this.flowAiChatHelpers?.testFlow(args), - attachedFiles: this.attachedFiles, - getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', - setUserInstructions: (instructions: string) => { - const prompts = getUserCustomPrompts() - if (instructions.trim()) { - prompts[AIMode.GLOBAL] = instructions - } else { - delete prompts[AIMode.GLOBAL] - } - setUserCustomPrompts(prompts) - this.rebuildGlobalSystemMessage() - } - } satisfies GlobalToolHelpers + this.configureGlobalMode() void this.refreshGlobalSkills() } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) @@ -992,6 +970,43 @@ export class AIChatManager { // Fetch the workspace's AI skills and, if GLOBAL mode is still active, rebuild // the system message so the next chat-loop iteration advertises them. Ignore // stale resolves so workspace changes cannot overwrite newer skills. + // Build the global-mode system message, tools, and helpers, layering on the + // pipeline surface when a /pipeline editor has registered helpers. Centralized + // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — + // each rebuild would otherwise drop the pipeline augmentation the others added. + private configureGlobalMode = () => { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + previewTools: this.isSessionChat, + skills: this.globalSkills + }) + const baseHelpers: GlobalToolHelpers = { + ...(this.isSessionChat ? { sessionId: this.sessionId } : {}), + testActiveFlow: async (args?: Record) => this.flowAiChatHelpers?.testFlow(args), + attachedFiles: this.attachedFiles, + getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', + setUserInstructions: (instructions: string) => { + const prompts = getUserCustomPrompts() + if (instructions.trim()) { + prompts[AIMode.GLOBAL] = instructions + } else { + delete prompts[AIMode.GLOBAL] + } + setUserCustomPrompts(prompts) + this.rebuildGlobalSystemMessage() + } + } + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + this.tools = [...globalToolsFor({ sessionPreview: this.isSessionChat }), ...pipelineTools] + this.helpers = { ...baseHelpers, pipeline } + } else { + this.tools = globalToolsFor({ sessionPreview: this.isSessionChat }) + this.helpers = baseHelpers + } + this.systemMessage = systemMessage + } + refreshGlobalSkills = async (workspace = get(workspaceStore) ?? '') => { const refreshId = ++this.globalSkillsRefreshId const skills = await loadWorkspaceSkills(workspace) @@ -1000,10 +1015,7 @@ export class AIChatManager { } this.globalSkills = skills if (this.mode === AIMode.GLOBAL) { - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { - previewTools: this.isSessionChat, - skills - }) + this.configureGlobalMode() } } @@ -1014,10 +1026,18 @@ export class AIChatManager { if (this.mode !== AIMode.GLOBAL) { return } - this.systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { + const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { previewTools: this.isSessionChat, skills: this.globalSkills }) + // Preserve the active pipeline-editor augmentation that configureGlobalMode + // adds — otherwise update_user_instructions (which calls this) would drop the + // /pipeline/ context + direct-draft/materialize guidance mid-session. + const pipeline = this.pipelineAiChatHelpers + if (pipeline) { + systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) + } + this.systemMessage = systemMessage } private expandGlobalSkillCommand = (instructions: string): string => { @@ -2132,7 +2152,7 @@ export class AIChatManager { moduleState && !moduleState.previewSuccess ? getStringError(moduleState.previewResult) : undefined, - getCode: () => module.value.type === 'rawscript' ? module.value.content : '', + getCode: () => (module.value.type === 'rawscript' ? module.value.content : ''), lang: module.value.language, path: module.id, ...editorRelated @@ -2176,6 +2196,28 @@ export class AIChatManager { } } + // Registered by the /pipeline editor while it is mounted. Rebuilds the global + // tool set so the pipeline tools appear (and disappear on unregister). Pipeline + // AI edits apply directly as drafts, so there is nothing to auto-accept. + // Returns a cleanup that tears the registration back down. + setPipelineHelpers = (pipelineHelpers: PipelineAIChatHelpers) => { + this.pipelineAiChatHelpers = pipelineHelpers + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + + return () => { + this.pipelineAiChatHelpers = undefined + untrack(() => { + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + }) + } + } + /** * Refresh cached datatables from the app helpers (async) * Creates one context element per table (not per datatable) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b85e271632..90dcb655f4 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -56,6 +56,7 @@ import { createInlineScriptSession } from '../flow/inlineScriptsUtils' import { getDatatableSdkReference, getFlowPrompt, + getPipelinePrompt, getRawAppPrompt, getResourcePrompt, getScriptPrompt @@ -138,7 +139,9 @@ const INSTRUCTION_SUBJECTS = [ ] as const satisfies readonly WorkspaceItemType[] // `datatable` is not a workspace item type, but the model can request the // datatable SDK reference (the wmill.datatable() runnable API) the same way. -const INSTRUCTION_SUBJECTS_EXTRA = ['datatable'] as const +// `pipeline` likewise isn't an item type — a data pipeline is a set of +// annotated scripts in a folder, so it gets authoring guidance, not a CRUD type. +const INSTRUCTION_SUBJECTS_EXTRA = ['datatable', 'pipeline'] as const const ALL_INSTRUCTION_SUBJECTS = [...INSTRUCTION_SUBJECTS, ...INSTRUCTION_SUBJECTS_EXTRA] as const const MAX_LIST_LIMIT = 100 type ActiveGlobalEditorType = Extract @@ -171,7 +174,7 @@ const scriptLangSchema = z.enum($ScriptLang.enum) const getInstructionsSchema = z.object({ subject: instructionSubjectSchema.describe( - 'What to get authoring instructions for: a workspace item type (script, flow, resource, app) or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' + 'What to get authoring instructions for: a workspace item type (script, flow, resource, app), "pipeline" for building a data pipeline (a DAG of annotated scripts wired by storage assets — NOT a flow), or "datatable" for the wmill.datatable() SQL SDK used inside runnables. Schedules, triggers, and variables don\'t need instructions — their tool schemas describe everything.' ), language: scriptLangSchema .optional() @@ -206,7 +209,9 @@ const updateUserInstructionsSchema = z.object({ .string() .min(1) .optional() - .describe("Required when operation is 'append': the instruction to add. Ignored for 'replace'."), + .describe( + "Required when operation is 'append': the instruction to add. Ignored for 'replace'." + ), old_string: z .string() .min(1) @@ -678,11 +683,13 @@ const deleteAppRunnableSchema = z.object({ const openPreviewSchema = z.object({ kind: z - .enum(['script', 'flow', 'raw_app']) + .enum(['script', 'flow', 'raw_app', 'pipeline']) .describe( - 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' + 'Item kind to preview. Use "raw_app" for code-based apps (created via init_app). Use "pipeline" to show the data-pipeline graph for a folder — here `path` is the folder name, not an item path. The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' ), - path: z.string().describe('Workspace path of the item to preview.') + path: z + .string() + .describe('Workspace path of the item to preview, or the folder name when kind is "pipeline".') }) const getPreviewStatusSchema = z.object({}) @@ -825,6 +832,7 @@ Rules: - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. +- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. @@ -833,6 +841,7 @@ Rules: previewTools ? ` - After writing or substantially editing a script / flow / app draft, show it via open_preview(kind, path) so the user sees the editor and live preview right next to the chat. First check whether it is already shown: if unsure, call get_preview_status. Only call open_preview (or offer to) when no preview is open or it is showing a different item — don't re-open a preview already showing the item you just edited. +- Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). - get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected.` : '' @@ -1608,6 +1617,10 @@ Datatables are workspace-scoped managed PostgreSQL databases. In chat, explore a ${getDatatableSdkReference(lang)}` } +function getPipelineInstructions(): string { + return getPipelinePrompt() +} + function getInstructions(subject: InstructionSubject, language?: ScriptLang): string { switch (subject) { case 'script': @@ -1620,6 +1633,8 @@ function getInstructions(subject: InstructionSubject, language?: ScriptLang): st return getAppInstructions() case 'datatable': return getDatatableInstructions(language) + case 'pipeline': + return getPipelineInstructions() } } @@ -1672,7 +1687,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( getInstructionsSchema, 'get_instructions', - 'Get authoring guidance for scripts, flows, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' + 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' ), fn: async ({ args, toolId, toolCallbacks }) => { const parsed = getInstructionsSchema.parse(args) @@ -2490,7 +2505,7 @@ function activeFlowTestFromCtx( export type OpenPreviewHandler = (req: { sessionId: string | undefined - kind: 'script' | 'flow' | 'raw_app' + kind: 'script' | 'flow' | 'raw_app' | 'pipeline' path: string }) => string @@ -2501,7 +2516,7 @@ export function setOpenPreviewHandler(handler: OpenPreviewHandler | undefined): } function openSessionPreview( - args: { kind: 'script' | 'flow' | 'raw_app'; path: string }, + args: { kind: 'script' | 'flow' | 'raw_app' | 'pipeline'; path: string }, sessionId: string | undefined ) { if (!openPreviewHandler) { diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts new file mode 100644 index 0000000000..4f3e31986d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts @@ -0,0 +1,171 @@ +import { describe, it, expect, vi } from 'vitest' + +// `../shared` transitively pulls in the monaco editor (and its CSS), which the +// node test environment can't load — mirror the sibling chat tests' stub. +vi.mock('monaco-editor', () => ({ editor: {} })) + +import { + pipelineTools, + getPipelinePromptSection, + type PipelineAIChatHelpers, + type PipelineContext +} from './core' +import type { ToolCallbacks } from '../shared' + +function toolByName(name: string) { + const tool = pipelineTools.find((t) => t.def.function.name === name) + if (!tool) throw new Error(`tool ${name} not found`) + return tool +} + +function noopCallbacks(): ToolCallbacks { + return { setToolStatus: () => {}, removeToolStatus: () => {} } +} + +const sampleContext: PipelineContext = { + folder: 'analytics', + mode: 'edit', + nodes: [ + { + path: 'f/analytics/orders', + language: 'bun', + unsaved: true, + writes: ['ducklake://main/orders'], + reads: [], + triggers: ['schedule'] + } + ], + assets: ['ducklake://main/orders'] +} + +function makeHelpers(overrides: Partial = {}): { + helpers: { pipeline: PipelineAIChatHelpers } + calls: Record +} { + const calls: Record = {} + const record = + (name: string, ret?: any) => + (...args: any[]) => { + ;(calls[name] ??= []).push(args) + return ret + } + const pipeline: PipelineAIChatHelpers = { + getPipelineContext: () => sampleContext, + getNodeBody: async (path: string) => { + calls.getNodeBody = [...(calls.getNodeBody ?? []), [path]] + return { language: 'bun', content: 'export async function main() { return 1 }' } + }, + proposeNode: async (input) => { + calls.proposeNode = [...(calls.proposeNode ?? []), [input]] + return { path: input.path } + }, + editNode: async (path, content) => { + calls.editNode = [...(calls.editNode ?? []), [path, content]] + }, + removeProposedNode: record('removeProposedNode'), + testNode: async () => 'job-123', + ...overrides + } + return { helpers: { pipeline }, calls } +} + +describe('pipeline tools', () => { + it('exposes the expected tool surface', () => { + expect(pipelineTools.map((t) => t.def.function.name).sort()).toEqual([ + 'build_pipeline_node', + 'edit_pipeline_node', + 'get_pipeline_graph', + 'read_pipeline_node', + 'remove_pipeline_node', + 'test_pipeline_node' + ]) + }) + + it('get_pipeline_graph returns the live context as JSON', async () => { + const { helpers } = makeHelpers() + const out = await toolByName('get_pipeline_graph').fn({ + args: {}, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(JSON.parse(out)).toMatchObject({ folder: 'analytics' }) + }) + + it('build_pipeline_node forwards to proposeNode and does not deploy', async () => { + const { helpers, calls } = makeHelpers() + const out = await toolByName('build_pipeline_node').fn({ + args: { + path: 'f/analytics/clean', + language: 'bun', + content: '// pipeline\nexport async function main() {}', + output_kind: 'ducklake' + }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.proposeNode?.[0]?.[0]).toMatchObject({ + path: 'f/analytics/clean', + language: 'bun', + outputKind: 'ducklake' + }) + expect(out).toContain('not deployed') + }) + + it('edit_pipeline_node reads then applies an exact find/replace', async () => { + const { helpers, calls } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\nconst y = 2\n' }) + }) + await toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'const x = 1', new_string: 'const x = 42' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(calls.editNode?.[0]?.[1]).toContain('const x = 42') + }) + + it('edit_pipeline_node surfaces a clear error when old_string is absent', async () => { + const { helpers } = makeHelpers({ + getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\n' }) + }) + await expect( + toolByName('edit_pipeline_node').fn({ + args: { path: 'f/analytics/orders', old_string: 'NOT THERE', new_string: 'x' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/was not found/) + }) + + it('mutation tools fail clearly when no pipeline editor is registered', async () => { + await expect( + toolByName('build_pipeline_node').fn({ + args: { path: 'f/a/b', language: 'bun', content: 'x' }, + workspace: 'w', + helpers: {}, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + ).rejects.toThrow(/No pipeline editor is open/) + }) + + it('test_pipeline_node requires confirmation', () => { + expect(toolByName('test_pipeline_node').requiresConfirmation).toBe(true) + }) +}) + +describe('getPipelinePromptSection', () => { + it('names the active folder and the direct-draft workflow', () => { + const section = getPipelinePromptSection(sampleContext) + expect(section).toContain('/pipeline/analytics') + expect(section).toContain('build_pipeline_node') + expect(section).toContain('directly as unsaved drafts') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts new file mode 100644 index 0000000000..f99bd0f210 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -0,0 +1,328 @@ +import { z } from 'zod' +import { $ScriptLang } from '$lib/gen/schemas.gen' +import type { ScriptLang } from '$lib/gen' +import { createToolDef, executeTestRun, findAndReplace, type Tool } from '../shared' +import type { PipelineOutputKind } from '$lib/components/assets/AssetGraph/pipelineTemplates' + +// ============================================================================ +// Pipeline AI chat tools. +// +// These tools extend the GLOBAL chat mode when the user is on a /pipeline/ +// editor (the page registers `PipelineAIChatHelpers` on the AIChatManager). They +// let the model read the live pipeline graph and BUILD/EDIT pipeline nodes +// (scripts annotated with `// pipeline`). Mutations don't deploy: they apply +// directly as an unsaved DRAFT on the canvas — the same way the flow/script +// editor applies AI edits — which the user then deploys. There is no separate +// approve/reject step: the draft IS the change. +// +// The pipeline tools are added on top of the full global tool set, so docs +// search, datatable SQL, and workspace-item tools are already available +// alongside them — this file only carries the pipeline-graph-specific surface. +// ============================================================================ + +/** Compact, model-facing summary of one node in the pipeline graph. */ +export type PipelineNodeSummary = { + path: string + language?: ScriptLang + /** Has an unsaved local edit (draft) not yet deployed. */ + unsaved: boolean + summary?: string + /** Asset URIs this node writes (its outputs). */ + writes: string[] + /** Asset URIs this node reads (its inputs). */ + reads: string[] + /** Declared `// on ` execution-DAG bindings (asset URIs or native kinds). */ + triggers: string[] +} + +/** Compact, model-facing snapshot of the whole pipeline graph. */ +export type PipelineContext = { + folder: string + mode: 'view' | 'edit' + nodes: PipelineNodeSummary[] + /** All storage assets referenced by the graph, as URIs. */ + assets: string[] +} + +/** + * Bridge the pipeline page registers on the AIChatManager. Reads expose the live + * graph; writes apply directly as unsaved drafts (never deploy). Kept intentionally + * small — the page owns the draft Map and canvas rendering. + */ +export interface PipelineAIChatHelpers { + getPipelineContext: () => PipelineContext + /** Read a node's source (the in-flight draft body if one exists, else deployed). */ + getNodeBody: (path: string) => Promise<{ language: ScriptLang; content: string } | undefined> + /** Create a brand-new pipeline node as an unsaved draft on the canvas. */ + proposeNode: (input: { + path: string + language: ScriptLang + content: string + outputKind?: PipelineOutputKind + }) => Promise<{ path: string }> + /** Replace an existing node's body, applied as an unsaved draft. */ + editNode: (path: string, content: string) => Promise + /** Discard the unsaved draft at a path (undo a build_pipeline_node). */ + removeProposedNode: (path: string) => Promise + /** Preview-run a node (draft body preferred). Returns the started job id. */ + testNode: (path: string, args?: Record) => Promise +} + +/** Helper bag the pipeline tools receive from the manager in global mode. */ +export type PipelineToolHelpers = { pipeline?: PipelineAIChatHelpers } + +function requirePipeline(helpers: PipelineToolHelpers): PipelineAIChatHelpers { + if (!helpers?.pipeline) { + throw new Error( + 'No pipeline editor is open. Pipeline tools only work on a /pipeline/ page in edit mode.' + ) + } + return helpers.pipeline +} + +const scriptLangSchema = z.enum($ScriptLang.enum) + +const outputKindSchema = z + .enum(['none', 'datatable', 'ducklake', 'materialize', 's3_parquet', 's3_object']) + .describe( + 'Kind of output asset this node materializes, used to seed the output edge on the canvas before the body is parsed: "materialize"/"ducklake" → a DuckLake table, "datatable" → a Postgres data table, "s3_parquet"/"s3_object" → an S3 file, "none" → side-effect only. Defaults to none.' + ) + +// ---------------------------------------------------------------------------- +// Read tools +// ---------------------------------------------------------------------------- + +const getPipelineGraphSchema = z.object({}) + +const getPipelineGraphToolDef = createToolDef( + getPipelineGraphSchema, + 'get_pipeline_graph', + "Read the live pipeline graph for the open /pipeline/ editor: its nodes (scripts), each node's language, asset reads/writes, declared triggers, and whether it has an unsaved draft edit. Call this before building or editing nodes so you reuse existing assets/paths and understand the current DAG." +) + +const readPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the pipeline node (script) to read.') +}) + +const readPipelineNodeToolDef = createToolDef( + readPipelineNodeSchema, + 'read_pipeline_node', + 'Read the full source of one pipeline node (its in-flight draft body if it has unsaved edits, otherwise the deployed body). Use before edit_pipeline_node so edits target the exact current text.' +) + +// ---------------------------------------------------------------------------- +// Mutation tools (apply directly as unsaved drafts; never deploy) +// ---------------------------------------------------------------------------- + +const buildPipelineNodeSchema = z.object({ + path: z + .string() + .describe( + "Workspace path for the new node, e.g. f//. Use the open pipeline's folder. Must not collide with an existing node." + ), + language: scriptLangSchema.describe( + 'Script language. SQL-shaped data work uses duckdb (DuckLake/S3) or postgresql (data tables); bun/python3 for general transforms.' + ), + content: z + .string() + .describe( + "Full script source. Start it with the `pipeline` annotation as a top-of-file comment in the LANGUAGE'S comment syntax — `-- pipeline` for SQL (duckdb/postgresql), `# pipeline` for python3/bash, `// pipeline` for bun/TS — to mark it a pipeline member; declare inputs the same way (e.g. `-- on `), and write outputs via the wmill SDK / SQL so the lineage edges are inferred. A `// pipeline` line in a SQL node is a syntax error. Read existing node bodies first to match conventions." + ), + output_kind: outputKindSchema.optional() +}) + +const buildPipelineNodeToolDef = createToolDef( + buildPipelineNodeSchema, + 'build_pipeline_node', + 'Build a NEW pipeline node. It is applied directly as an unsaved draft on the canvas (a dashed node wired by its parsed asset reads/writes) — it does NOT deploy; the user deploys it. Prefer this over editing for new scripts.', + { strict: false } +) + +const editPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to edit.'), + old_string: z.string().min(1).describe("Exact text to find in the node's current source."), + new_string: z.string().describe('Replacement text.'), + replace_all: z + .boolean() + .optional() + .default(false) + .describe( + 'When true, replace every exact match. When false, old_string must match exactly once.' + ) +}) + +const editPipelineNodeToolDef = createToolDef( + editPipelineNodeSchema, + 'edit_pipeline_node', + 'Edit an existing pipeline node by exact find/replace. The result is applied directly as an unsaved draft (does NOT deploy). Call read_pipeline_node first to get the exact current text.', + { strict: false } +) + +const removePipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node whose unsaved draft should be discarded.') +}) + +const removePipelineNodeToolDef = createToolDef( + removePipelineNodeSchema, + 'remove_pipeline_node', + 'Discard the unsaved draft at a path (undo a build_pipeline_node). Only affects the in-flight draft — to delete a deployed node, ask the user to do it on the canvas.' +) + +const testPipelineNodeSchema = z.object({ + path: z.string().describe('Workspace path of the node to preview-run.'), + args: z + .record(z.string(), z.any()) + .nullable() + .optional() + .describe('Arguments to pass to the script. Omit or pass null when none are needed.') +}) + +const testPipelineNodeToolDef = createToolDef( + testPipelineNodeSchema, + 'test_pipeline_node', + 'Preview-run one pipeline node (using its draft body when unsaved) and return the result/logs, without deploying. Requires user confirmation before it runs.', + { strict: false } +) + +// ---------------------------------------------------------------------------- +// Tool set +// ---------------------------------------------------------------------------- + +export const pipelineTools: Tool[] = [ + { + def: getPipelineGraphToolDef, + fn: async ({ helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + toolCallbacks.setToolStatus(toolId, { content: 'Reading pipeline graph...' }) + const ctx = pipeline.getPipelineContext() + toolCallbacks.setToolStatus(toolId, { + content: `Read pipeline graph (${ctx.nodes.length} node${ctx.nodes.length === 1 ? '' : 's'})`, + result: 'Success' + }) + return JSON.stringify(ctx, null, 2) + } + }, + { + def: readPipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = readPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Reading node '${path}'...` }) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Read node '${path}'`, result: 'Success' }) + return JSON.stringify({ path, language: node.language, content: node.content }) + } + }, + { + def: buildPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, language, content, output_kind } = buildPipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Building node '${path}'...` }) + await pipeline.proposeNode({ + path, + language: language as ScriptLang, + content, + outputKind: output_kind as PipelineOutputKind | undefined + }) + toolCallbacks.setToolStatus(toolId, { + content: `Added draft node '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' added as an unsaved draft on the canvas. It is not deployed — the user deploys it.` + } + }, + { + def: editPipelineNodeToolDef, + streamArguments: true, + showDetails: true, + showFade: true, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, old_string, new_string, replace_all } = editPipelineNodeSchema.parse(args) + const node = await pipeline.getNodeBody(path) + if (!node) { + return `No pipeline node found at '${path}'. Call get_pipeline_graph to list the available nodes.` + } + toolCallbacks.setToolStatus(toolId, { content: `Editing node '${path}'...` }) + const updated = findAndReplace( + node.content, + old_string, + new_string, + replace_all ?? false, + 'node source' + ) + await pipeline.editNode(path, updated) + toolCallbacks.setToolStatus(toolId, { + content: `Edited draft '${path}'`, + result: 'Success' + }) + return `Pipeline node '${path}' updated as an unsaved draft on the canvas (not deployed).` + } + }, + { + def: removePipelineNodeToolDef, + fn: async ({ args, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path } = removePipelineNodeSchema.parse(args) + toolCallbacks.setToolStatus(toolId, { content: `Discarding draft '${path}'...` }) + await pipeline.removeProposedNode(path) + toolCallbacks.setToolStatus(toolId, { + content: `Discarded draft '${path}'`, + result: 'Success' + }) + return `Discarded the unsaved draft at '${path}'.` + } + }, + { + def: testPipelineNodeToolDef, + requiresConfirmation: true, + confirmationMessage: 'Run pipeline node', + showDetails: true, + autoCollapseDetails: false, + fn: async ({ args, workspace, helpers, toolId, toolCallbacks }) => { + const pipeline = requirePipeline(helpers) + const { path, args: runArgs } = testPipelineNodeSchema.parse(args) + return executeTestRun({ + jobStarter: async () => { + const jobId = await pipeline.testNode(path, runArgs ?? undefined) + if (!jobId) { + throw new Error(`Could not start a run for node '${path}'.`) + } + return jobId + }, + workspace, + toolCallbacks, + toolId, + startMessage: `Starting run of '${path}'...`, + contextName: 'script' + }) + } + } +] + +/** + * Pipeline-specific guidance appended to the global system prompt when a + * /pipeline editor is open. Describes the annotation model and the direct-draft + * workflow so the model uses the pipeline tools rather than the generic + * write_script draft tools. + */ +export function getPipelinePromptSection(ctx: PipelineContext): string { + return ` + +Data Pipeline editor (ACTIVE): +- The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. +- Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. +- Build new nodes with build_pipeline_node and edit existing ones with edit_pipeline_node. These apply directly as unsaved drafts on the canvas (like the flow/script editor applies AI edits) — they DO NOT deploy. There is no separate Accept/Reject step. Prefer these over the generic write_script/edit_script draft tools while a pipeline is open. +- Reuse existing asset paths from the graph when wiring a downstream node to an upstream one (read the upstream's write asset, then \`// on\` that same URI). +- Only deploy when the user explicitly asks; the user deploys drafts from the canvas.` +} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index b6ae577d4b..086760ac6e 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -13,6 +13,7 @@ type ListableRawApp } from '$lib/gen' import { resource } from 'runed' + import { getDraftItems } from '$lib/workspaceDrafts.svelte' import { userStore, workspaceStore } from '$lib/stores' import type uFuzzy from '@leeoniya/ufuzzy' import { @@ -73,23 +74,44 @@ type TableApp = TableItem type TableRawApp = TableItem - // Folders with ≥1 pipeline script (auto_kind='pipeline'). Used by - // TreeView to surface a "Pipeline" entry inside those folders. Cheap - // thanks to the partial index on script.auto_kind. + // Folders that are data pipelines, surfaced as their own "Pipeline" entry + // (the member scripts are folded into it, not listed individually). Two + // sources: deployed pipelines (folders with ≥1 `auto_kind='pipeline'` script, + // cheap via the partial index) AND bundle-phase pipelines that only exist as a + // `data_pipeline` draft so far — so a pipeline shows up the moment its first + // node is drafted, before anything is deployed. let pipelineFoldersRes = resource( () => $workspaceStore, async (ws) => { if (!ws) return new Set() + const folders = new Set() try { - const rows = await AssetService.listPipelineFolders({ workspace: ws }) - return new Set(rows.map((r) => r.folder)) + for (const r of await AssetService.listPipelineFolders({ workspace: ws })) + folders.add(r.folder) } catch { - // Decorative tree entry — degrade to "no pipelines" on failure. - return new Set() + // Decorative entry — degrade gracefully on failure. } + try { + for (const d of await getDraftItems(ws)) { + if (d.kind !== 'data_pipeline') continue + const m = d.path.match(/^f\/([^/]+)\/data_pipeline$/) + if (m) folders.add(m[1]) + } + } catch { + // Drafts unavailable — show deployed pipelines only. + } + return folders } ) - let pipelineFolders = $derived(pipelineFoldersRes.current ?? new Set()) + // Folders of pipeline-member scripts present in the current listing (captured + // in loadScripts before they're filtered out). Unioned in so a folder whose + // only pipeline node is a never-deployed `// pipeline` script draft — not in + // listPipelineFolders (deployed-only) nor a `data_pipeline` bundle — still gets + // a pipeline entry instead of vanishing. + let pipelineMemberFolders = $state(new Set()) + let pipelineFolders = $derived( + new Set([...(pipelineFoldersRes.current ?? []), ...pipelineMemberFolders]) + ) let scripts: TableScript[] | undefined = $state() let flows: TableFlow[] | undefined = $state() @@ -115,12 +137,26 @@ withoutDescription: true }) - scripts = loadedScripts.map((script: Script) => { - return { - canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, - ...script - } - }) + // Pipeline-member scripts (`auto_kind='pipeline'`) are represented by their + // pipeline's entry, not listed individually — but capture their folders so + // the pipeline entry still surfaces (incl. a members-only / draft-only folder). + const memberFolders = new Set() + scripts = loadedScripts + .filter((script: Script) => { + if (script.auto_kind === 'pipeline') { + const m = script.path.match(/^f\/([^/]+)\//) + if (m) memberFolders.add(m[1]) + return false + } + return true + }) + .map((script: Script) => { + return { + canWrite: canWrite(script.path, script.extra_perms, $userStore) && !$userStore?.operator, + ...script + } + }) + pipelineMemberFolders = memberFolders loading = false } @@ -246,6 +282,25 @@ : undefined ) let filterUserFolders = $state(getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true') + + // Pipeline entries are rendered independently of the item list, so apply the + // same gates the items get — otherwise a pipeline would still show under the + // Flows/Apps tabs, in the archived view, under a label filter, or outside a + // selected owner. Pipelines are script-based units always at `f/`, so + // kind=script and the user-folder toggle always include them; kind=flow/app, + // archived, a label filter (pipelines carry no labels), and a non-matching + // owner exclude them. + let visiblePipelineFolders = $derived.by(() => { + if (archived) return new Set() + if (itemKind !== 'all' && itemKind !== 'script') return new Set() + if (labelFilter != undefined) return new Set() + if (ownerFilter == undefined) return pipelineFolders + return new Set( + [...pipelineFolders].filter( + (f) => `f/${f}` === ownerFilter || `f/${f}`.startsWith(ownerFilter + '/') + ) + ) + }) let includeWithoutMain = $state( getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) ? getLocalSetting(INCLUDE_WITHOUT_MAIN_SETTING_NAME) == 'true' @@ -826,14 +881,17 @@ {#each new Array(6) as _} {/each} - {:else if filteredItems.length === 0} + {:else if filteredItems.length === 0 && (filter !== '' || visiblePipelineFolders.size === 0)} + {:else if treeView} loadScripts(includeWithoutMain)} on:flowChanged={loadFlows} @@ -850,7 +908,7 @@ {:else}
{#if filter === ''} - {#each [...pipelineFolders].sort() as folder (folder)} + {#each [...visiblePipelineFolders].sort() as folder (folder)} i && 'folderName' in i + // Hidden while searching: pipelines aren't part of the text filter (the list + // view hides their rows on a query too), so a folder matching the search + // shouldn't surface an unrelated Pipeline row. let hasPipeline = $derived( - depth === 0 && isFolderItem(item) && (pipelineFolders?.has(item.folderName) ?? false) + depth === 0 && + !isSearching && + isFolderItem(item) && + (pipelineFolders?.has(item.folderName) ?? false) ) const isFolder = isFolderItem diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index 59adf1aaa9..ef5cf25d58 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -24,7 +24,42 @@ let groupedItems: ReturnType | 'loading' = $state('loading') $effect(() => { items - untrack(() => (groupedItems = groupItems(items))) + pipelineFolders + isSearching + untrack(() => { + const grouped = groupItems(items) + // Ensure every pipeline folder is present at the top level so its + // "Pipeline" entry shows even when it has no listed items — a bundle-phase + // pipeline (only a draft so far) or a folder whose only scripts are + // pipeline members (folded into the pipeline, hidden from the list). + // Skip while searching: pipelines aren't part of the text filter (list view + // hides them on `filter !== ''`), so injecting them would surface unrelated + // folders in the results. + if (!isSearching) { + const present = new Set( + grouped + .filter((g) => 'folderName' in g) + .map((g) => (g as { folderName: string }).folderName) + ) + // Insert each missing pipeline folder among the existing folders in name + // order — `groupItems` already sorts user groups first then folders + // alphabetically, so inserting before the first greater-named folder + // keeps that ordering (rather than prepending out of order). + for (const folderName of [...(pipelineFolders ?? [])] + .filter((f) => !present.has(f)) + .sort()) { + const item = { folderName, items: [] } + const idx = grouped.findIndex( + (g) => + 'folderName' in g && + (g as { folderName: string }).folderName.localeCompare(folderName) > 0 + ) + if (idx < 0) grouped.push(item) + else grouped.splice(idx, 0, item) + } + } + groupedItems = grouped + }) }) diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 19e71530e6..3e631a55e9 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -505,12 +505,16 @@ time: new Date(x.edited_at).getTime(), search_id: x.path })), - ...scripts.map((x) => ({ - ...x, - type: 'script' as 'script', - time: new Date(x.created_at).getTime(), - search_id: x.path - })), + // Pipeline-member scripts (`auto_kind='pipeline'`) are reached through + // their pipeline, not searched individually. + ...scripts + .filter((x) => x.auto_kind !== 'pipeline') + .map((x) => ({ + ...x, + type: 'script' as 'script', + time: new Date(x.created_at).getTime(), + search_id: x.path + })), ...apps.map((x) => ({ ...x, type: 'app' as 'app', diff --git a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte index 0716398131..0b529b2056 100644 --- a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte @@ -1,7 +1,7 @@ + +
+
+ + f/{path} + · data pipeline +
+
+ + {#if graphRes.loading && !graphRes.current && pe.drafts.size === 0} +
+ + Loading pipeline… +
+ {:else if graphRes.error && pe.drafts.size === 0} +
+ Failed to load pipeline: {graphRes.error.message} +
+ {:else} + + runNode(path, args)} + canRunByPath + onTestStateChange={(running) => { + const openPath = pe.openScriptPath + if (running && openPath) { + activeRunnable = { kind: 'script', path: openPath } + activeRunnables.arm(`script:${openPath}`) + activeRunnableJobId = undefined + } else if (!running && activeRunnable?.path === openPath) { + // Only clear the hint for the script the pane just finished — a + // canvas per-node run of a different script keeps its own hint. + activeRunnable = undefined + activeRunnableJobId = undefined + } + }} + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onSelect={handleCanvasSelect} + onDraftSaved={afterSaved} + onPersistedSaved={afterSaved} + onScriptRemoved={async (removedPath) => { + pe.forgetPath(removedPath) + await graphRes.refetch() + }} + onScriptRenamed={async (oldPath, newPath) => { + // Repoint the selection so the canvas follows the renamed node instead + // of staying on the now-gone old path until an unrelated refetch. + if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { + pe.selection = { ...pe.selection, path: newPath } + } + await graphRes.refetch() + }} + onDiscard={() => { + if (pe.activeDraftPath) pe.discardDraft(pe.activeDraftPath) + }} + onClose={() => { + pe.selection = undefined + pe.activeDraftPath = undefined + pe.clearLiveOverlays() + }} + /> + {/if} +
+
+ + + graphRes.refetch()} +/> diff --git a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte index dc1a088c53..77c5173813 100644 --- a/frontend/src/lib/components/sessions/SessionItemNotFound.svelte +++ b/frontend/src/lib/components/sessions/SessionItemNotFound.svelte @@ -3,7 +3,11 @@ import type { WorkspaceItem, WorkspaceItemKind } from '$lib/components/workspacePicker' import type { SessionTarget } from './sessionState.svelte' - const KIND_NOT_FOUND_LABEL: Record = { + // `pipeline` targets never hit this component (they aren't slot-loaded, so they + // can't 404 through SessionEditorTarget) — exclude it from the kinds here. + type NotFoundKind = Exclude + + const KIND_NOT_FOUND_LABEL: Record = { flow: 'Flow', script: 'Script', raw_app: 'Raw app' @@ -14,7 +18,7 @@ path, onNavigate }: { - kind: SessionTarget['kind'] + kind: NotFoundKind path: string onNavigate?: (item: WorkspaceItem) => void } = $props() diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index 027089d4ae..f62aa07926 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -27,6 +27,7 @@ import FlowEditorView from './FlowEditorView.svelte' import ScriptEditorView from './ScriptEditorView.svelte' import RawAppEditorView from './RawAppEditorView.svelte' + import PipelineEditorView from './PipelineEditorView.svelte' import SessionWorkspaceBar from './SessionWorkspaceBar.svelte' import SessionForkBar from './SessionForkBar.svelte' import SessionDraftBar from './SessionDraftBar.svelte' @@ -261,7 +262,8 @@ {@const hasTarget = session.target?.kind === 'flow' || session.target?.kind === 'script' || - session.target?.kind === 'raw_app'} + session.target?.kind === 'raw_app' || + session.target?.kind === 'pipeline'} {@const hasEditor = mountEditor && hasTarget && editorVisible} {#snippet inputPreface()} @@ -469,6 +471,13 @@ onNavigate={pickEditorTarget} isActiveSession={sessionState.currentSessionId === sessionId} /> + {:else if session.target.kind === 'pipeline'} + {/if}
diff --git a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte index c4099edd91..e6b4e5fd2f 100644 --- a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte @@ -15,6 +15,11 @@ /** Summary supplied by the data source. Preferred over the one derived * from the loaded diff value, and shown before that value loads. */ summary?: string + /** Explicit unique row identity, overriding the default `kind/path`. For a + * row whose `kind/path` isn't unique on its own (a pipeline-bundle node + * shares `script/` with a standalone script draft at the same path) + * while `path` must stay the real edit/display target. */ + key?: string } @@ -110,6 +115,7 @@ // e.g. a runnable rendered as `script` at `/runnables/foo` vs a real // script literally at that path. Prefix synthetic items so the {#each} key, // load cache, row id and nav identity never collide with a real DiffRow. + if ('key' in d && d.key) return d.key return ('appPath' in d ? 'rawapp:' : '') + `${d.kind}/${d.path}` } diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index d7d9f77630..f8a9e1045d 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -1,6 +1,7 @@ import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { get } from 'svelte/store' import { AIChatManager, AIMode } from '$lib/components/copilot/chat/AIChatManager.svelte' +import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' import { initFlow } from '$lib/components/flows/flowStore.svelte' import { AppService, @@ -79,6 +80,9 @@ export type SessionTargetKind = 'flow' | 'script' | 'raw_app' export interface SessionRuntime { readonly sessionId: string readonly manager: AIChatManager + // Pipeline target state — persists across editor hide/show (the pane unmounts + // on hide, so this can't be component-local) and across session switches. + readonly pipelineEditorState: PipelineEditorState // Kind-agnostic accessor over the per-kind load slots, for consumers (the // editor-target gate) that only need load state and not the typed store. slot(kind: SessionTargetKind): LoadSlot @@ -307,6 +311,12 @@ function createRuntime(session: Session): SessionRuntime { const rawApp: { val: SessionRuntime['rawApp']['val'] } = $state({ val: undefined }) const savedRawApp: { val: SessionRuntime['savedRawApp']['val'] } = $state({ val: undefined }) const rawAppSlot: LoadSlot = $state({ loadedPath: undefined, loading: false, notFound: false }) + + // Pipeline target state lives on the runtime (not the PipelineEditorView + // component) so the in-session drafts survive hide/show of the editor pane — + // the pane unmounts on hide, and a component-local store would be discarded. + const pipelineEditorState = new PipelineEditorState() + let runtimeLogRequester: RawAppRuntimeLogRequester | undefined = undefined let appRunsProvider: RawAppRunsProvider | undefined = undefined @@ -344,6 +354,7 @@ function createRuntime(session: Session): SessionRuntime { slot(kind: SessionTargetKind): LoadSlot { return kind === 'flow' ? flowSlot : kind === 'script' ? scriptSlot : rawAppSlot }, + pipelineEditorState, flowStore, flowStateStore, savedFlow, diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index e3d9e620f6..4e12d8622f 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -31,14 +31,18 @@ import type HistoryManager from '$lib/components/copilot/chat/HistoryManager.sve import { onUserChange } from '$lib/userScopedStorage' // Kinds the in-session editor pane can host. Legacy drag-and-drop apps are -// intentionally not previewable — only code-based 'raw_app' apps are. -export type SessionTarget = { kind: 'flow' | 'script' | 'raw_app'; path: string } +// intentionally not previewable — only code-based 'raw_app' apps are. A +// 'pipeline' target's `path` is the folder name (not a workspace item path): +// it hosts the data-pipeline graph editor for that folder, which uses its own +// fetch/draft model rather than the single-item load slots the other kinds share. +export type SessionTarget = { kind: 'flow' | 'script' | 'raw_app' | 'pipeline'; path: string } // Useful for filtering dropdowns / pickers to "items the side panel can open". export const EDITOR_TARGET_KINDS: ReadonlySet = new Set([ 'flow', 'script', - 'raw_app' + 'raw_app', + 'pipeline' ]) // Lifecycle status for a fork session. Git-parallel: diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 3056245e51..7933ef6d5e 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -4,7 +4,7 @@ import { page } from '$app/state' import Button from '$lib/components/common/button/Button.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import AssetGraphCanvas from '$lib/components/assets/AssetGraph/AssetGraphCanvas.svelte' + import PipelineGraphEditor from '$lib/components/assets/AssetGraph/PipelineGraphEditor.svelte' import { useActiveRunnableIds, isActiveEvent @@ -15,9 +15,7 @@ RunStatus } from '$lib/components/assets/AssetGraph/activeRunnables.svelte' import { usePipelineHistory } from '$lib/components/assets/AssetGraph/pipelineHistory.svelte' - import PipelineEventLog from '$lib/components/assets/AssetGraph/PipelineEventLog.svelte' import PipelineActivityPanel from '$lib/components/assets/AssetGraph/PipelineActivityPanel.svelte' - import AssetGraphDetailsPane from '$lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte' import PipelinePickerModal from '$lib/components/assets/AssetGraph/PipelinePickerModal.svelte' import { extractWrites, @@ -31,11 +29,7 @@ PipelineMode } from '$lib/components/assets/AssetGraph/types' import PipelineModeToggle from '$lib/components/assets/AssetGraph/PipelineModeToggle.svelte' - import { - parsePipelineAnnotations, - type ColumnLineage, - type PipelineAnnotations - } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' + import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' import { buildColumnGraph, type ColumnLineageGraph @@ -68,9 +62,11 @@ type PipelineOutputKind, type DraftTriggerSource } from '$lib/components/assets/AssetGraph/pipelineTemplates' - import { decodeState, encodeState } from '$lib/utils' - import { DraftService } from '$lib/gen' - import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { + createPipelineAiHelpers, + type PipelineDraft + } from '$lib/components/assets/AssetGraph/pipelineAiHelpers' + import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' import AutosaveIndicator from '$lib/components/AutosaveIndicator.svelte' import { onMount, tick, untrack } from 'svelte' import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' @@ -98,14 +94,12 @@ type ScriptLang } from '$lib/gen' import { resource } from 'runed' - import { Pane, Splitpanes } from 'svelte-splitpanes' import { emptySchema, sendUserToast } from '$lib/utils' import type { Schema } from '$lib/common' import { beforeNavigate, goto } from '$app/navigation' import { fade } from 'svelte/transition' import { twMerge } from 'tailwind-merge' import Popover from '$lib/components/meltComponents/Popover.svelte' - import HideButton from '$lib/components/apps/editor/settingsPanel/HideButton.svelte' import { inferArgs, inferAssets } from '$lib/infer' import PipelineTriggerEditors from '$lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte' @@ -115,7 +109,33 @@ const DATA_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] let folder = $derived(page.params.folder as string) - let selection = $state(undefined) + + // Externalized editor state (drafts, live overlays, selection), shared with + // the in-session pipeline preview via PipelineEditorState. Referenced as + // `pe.*` throughout; the persistence / graph / run logic below stays here. + const pe = new PipelineEditorState() + + // The in-app folder switcher navigates same-route (`/pipeline/`), which + // reuses this page component — nothing remounts. Mirror the session preview's + // retarget guard: reset the editor state on a folder change so folder A's drafts + // don't display under B (and aren't autosaved into B's bundle), and so + // hydratedFromDb flips back to false and B's draft bundle re-hydrates. + $effect(() => { + const f = folder + untrack(() => { + if (pe.folder !== f) { + if (pe.folder !== undefined) { + pe.reset() + // Re-scope the Global chat's pipeline prompt to the new folder. The + // helper methods already read the reactive folder, but the system + // message string was built for the old one and is only rebuilt when + // Global mode is reconfigured — so rebuild it here. + aiChatManager.rebuildGlobalSystemMessage() + } + pe.folder = f + } + }) + }) // Page mode, URL-addressable via `?mode=`. No param = view (the // default): deployed-only graph focused on past/live executions. @@ -129,10 +149,10 @@ // selection / open draft / hidden-pane state so view opens on Activity // rather than a stale details pane (mirrors toggleActivity's show path). if (m === 'view' && mode === 'edit') { - selection = undefined - activeDraftPath = undefined + pe.selection = undefined + pe.activeDraftPath = undefined panelHidden = false - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } const url = new URL(page.url) if (m === 'view') url.searchParams.delete('mode') @@ -166,346 +186,44 @@ // output asset, and they all render on the graph simultaneously. // Saving removes a draft from the map; closing the pane keeps it so the // user can come back to it. - // Counter-based id source — sufficient for "stable across renames in - // this session"; doesn't need to survive a reload. (We have crypto. - // randomUUID() too but a short numeric id keeps localStorage tidy.) - let nextDraftLocalIdCounter = 0 - function newDraftLocalId(): string { - nextDraftLocalIdCounter += 1 - return `d${nextDraftLocalIdCounter}-${Date.now()}` - } + // The draft shape + draft Map + activeDraftPath now live in the shared + // PipelineEditorState (`pe`). `Draft` aliases the store's type so existing + // annotations keep working. + type Draft = PipelineDraft - type Draft = { - // Stable per-draft identifier, generated on first create and - // preserved across renames. Used to track concurrent deploys (a - // fast double-rename otherwise fires two saves that each leave a - // persisted script behind — the latest deploy archives the prior - // one keyed on this id). - localId: string - script: Script - // Undefined when the user picked `outputKind === 'none'` — the draft - // has no auto-generated output asset, so the graph overlay skips - // synthesizing a write edge for it. - outputAsset?: { kind: AssetKind; path: string } - // Inferred body writes from the last time this draft was open in - // the details pane. Captured on transition (selection change / - // pane close) so the canvas keeps showing the user's renamed - // outputs after they've clicked away. Falls back to outputAsset - // when undefined (initial state or parser miss). - outputAssets?: Array<{ kind: AssetKind; path: string }> - } - let drafts = $state>(new Map()) - - // Which draft (if any) is currently open in the details pane. When - // undefined and `selection` is set, the pane shows the persisted - // selection's script. Never both at once. - let activeDraftPath = $state(undefined) - - // Splitpanes sizes: bound so user-resized widths persist when the - // details pane is hidden + re-shown, or when switching between draft - // and persisted selections (previously the panes were sized inline, - // which made the right pane jump width when activeDraft was set - // since the left was hardcoded to 100%). When the right pane - // unmounts, svelte-splitpanes does NOT auto-stretch the remaining - // pane — the bound `leftPaneSize` stays at its last value and the - // other 40% renders as blank space. We explicitly set leftPaneSize - // to 100 in that state via a $derived effect below, and restore the - // stored split when the pane comes back. - let leftPaneSize = $state(60) - let rightPaneSize = $state(40) - let storedRightPaneSize = $state(40) + // Splitpane sizing + details-pane-open derivation live inside PipelineGraphEditor. // Explicit hide flag — keeps `selection` / `activeDraftPath` intact // so re-opening the pane re-uses them. Mirrors AppEditor's // hideRightPanel/showRightPanel pattern. let panelHidden = $state(false) - // Edit mode opens the pane only for a selection/draft; view mode - // keeps it open permanently — it shows the activity panel when nothing - // is selected and swaps to the details pane on node select. - let detailsPaneOpen = $derived( - mode === 'edit' - ? (selection != undefined || activeDraftPath != undefined) && !panelHidden - : !panelHidden - ) // View mode's idle pane is the activity feed; once a node is selected // (or the pane hidden) the only ways back were the pane's X and the // floating hide toggle — neither is named. The top-bar Activity toggle // is the explicit affordance: shows the feed from any state, hides the // pane when the feed is already showing. let activityShowing = $derived( - mode === 'view' && !panelHidden && selection == undefined && activeDraftPath == undefined + mode === 'view' && !panelHidden && pe.selection == undefined && pe.activeDraftPath == undefined ) function toggleActivity() { if (activityShowing) { panelHidden = true } else { - selection = undefined - activeDraftPath = undefined + pe.selection = undefined + pe.activeDraftPath = undefined panelHidden = false // Same reset as the pane's close button — clears the live // annotation overlay of whichever script was open. - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } } - $effect(() => { - if (detailsPaneOpen) { - // Pane visible: reapply the stored split. We don't depend on - // rightPaneSize here (only stored) so user resizes via the - // splitter handle aren't immediately overridden. - const restore = storedRightPaneSize - untrack(() => { - rightPaneSize = restore - leftPaneSize = 100 - restore - }) - } else { - // About to hide. Stash the current right size so the next show - // restores it, then expand the left pane to fill — splitpanes - // won't do this automatically when a Pane unmounts. - untrack(() => { - if (rightPaneSize > 0) storedRightPaneSize = rightPaneSize - leftPaneSize = 100 - }) - } - }) - - // All of this folder's in-flight drafts live in ONE per-user DB draft - // (typ `data_pipeline`) keyed at the folder, so they sync across devices - // and surface in the global drafts list — replacing the prior - // browser-only blob. The `f//...` path drives the backend's - // folder-write access check. localStorage is kept as a synchronous crash - // mirror (no size cap, survives a hard close inside the debounce window) - // but is only ever READ for the one-time import below; the DB is the - // source of truth on load. - const PIPELINE_DRAFT_KIND = 'data_pipeline' as const + // Draft autosave (the data_pipeline DraftService bundle) lives inside + // PipelineGraphEditor now; the route just supplies its path to the indicator. let pipelineDraftPath = $derived(`f/${folder}/data_pipeline`) - let storageKey = $derived(`pipeline-${folder}`) - type PipelineDraftBundle = { drafts: Array<[string, Draft]>; activeDraftPath?: string } - // Gate the persist effect until the initial DB load resolves so empty - // pre-hydration state can't clobber the server copy. `lastPersistedBundle` - // holds the last value we pushed so an unchanged re-render (and the - // just-loaded value itself) isn't re-POSTed. - let draftsHydrated = $state(false) - let lastPersistedBundle: string | undefined = undefined - // True once a bundle was restored from the DB on load — drives the - // AutosaveIndicator's one-shot "Loaded from draft" hint. - let loadedFromDbDraft = $state(false) - - function restoreBundle(bundle: PipelineDraftBundle) { - if (Array.isArray(bundle.drafts)) { - const loaded = new Map() - for (const entry of bundle.drafts) { - if (entry && typeof entry[0] === 'string' && entry[1]?.script) { - const d = entry[1] as Draft - // Backfill localId for state persisted by older builds. - if (typeof d.localId !== 'string' || d.localId === '') { - d.localId = newDraftLocalId() - } - loaded.set(entry[0], d) - } - } - if (loaded.size > 0) drafts = loaded - } - if (typeof bundle.activeDraftPath === 'string') { - activeDraftPath = bundle.activeDraftPath - } - } - - // The pre-DB localStorage blob, for the one-time migration when the user - // has no DB draft yet. - function readLocalBundle(): PipelineDraftBundle | undefined { - if (typeof localStorage === 'undefined') return undefined - const raw = localStorage.getItem(`pipeline-${folder}`) - if (!raw) return undefined - try { - const state = decodeState(raw) - if (state && (Array.isArray(state.drafts) || typeof state.activeDraftPath === 'string')) { - return { - drafts: Array.isArray(state.drafts) ? state.drafts : [], - activeDraftPath: - typeof state.activeDraftPath === 'string' ? state.activeDraftPath : undefined - } - } - } catch (e) { - console.warn('failed to read local pipeline state', e) - } - return undefined - } - - onMount(() => { - void hydrateDrafts() - }) - - async function hydrateDrafts() { - const ws = $workspaceStore - const path = pipelineDraftPath - try { - let bundle: PipelineDraftBundle | undefined - let serverSavedAt: string | undefined - if (ws) { - const row = await DraftService.getOwnDraft({ - workspace: ws, - kind: PIPELINE_DRAFT_KIND, - path - }) - if (row?.value) { - bundle = row.value as PipelineDraftBundle - serverSavedAt = row.created_at - loadedFromDbDraft = true - } - } - // One-time migration: no DB draft yet, but an older build left a - // localStorage blob — adopt it and let the persist effect push it up. - let migratedFromLocal = false - if (!bundle) { - const local = readLocalBundle() - if (local) { - bundle = local - migratedFromLocal = true - } - } - if (bundle) restoreBundle(bundle) - // Seed the conflict baseline: server timestamp when loaded from the - // DB, none otherwise (first save omits last_sync → backend first-push - // branch). A local migration counts as "nothing server-side yet". - UserDraftDbSyncer.recordRemoteSync( - { workspace: ws ?? '', itemKind: PIPELINE_DRAFT_KIND, path }, - migratedFromLocal ? undefined : serverSavedAt - ) - // Record what we loaded so the first persist run is a no-op — UNLESS - // we migrated from localStorage, which must push to the DB once. - if (!migratedFromLocal) { - lastPersistedBundle = bundle ? JSON.stringify(bundle) : undefined - } - } catch (e) { - console.warn('failed to load pipeline drafts', e) - } finally { - draftsHydrated = true - } - } - - // Persist on change: debounced DB sync (UserDraftDbSyncer handles the - // debounce + optimistic-concurrency) plus a synchronous localStorage - // mirror for crash recovery before the first DB confirm. - $effect(() => { - // Track deps explicitly so Svelte 5 re-runs on mutation. - // For the active draft, also snapshot the latest live body writes - // at serialize time. Without this, edits made since the last - // pane-transition (the cleanup that calls onDraftPersist) are - // lost on reload — the draft restores with stale `outputAssets` - // and the graph drops the corresponding edges until the user - // re-opens the draft and types something new. - const liveWritesSnapshot = - liveBodyAssets.scriptPath != undefined && drafts.has(liveBodyAssets.scriptPath) - ? extractWrites(liveBodyAssets.assets) - : undefined - const liveWritesPath = liveBodyAssets.scriptPath - // Live editor buffer for the open draft. `onDraftPersist` only commits - // content into the Map on pane teardown, so without this overlay the - // autosave (and a crash reload) would lag a whole editing session behind. - const liveContentPath = - liveContent.scriptPath != undefined && drafts.has(liveContent.scriptPath) - ? liveContent.scriptPath - : undefined - const liveContentValue = liveContent.content - const serialized = Array.from(drafts.entries()).map(([p, d]) => { - const outputAssets = - liveWritesSnapshot != undefined && liveWritesPath === p - ? liveWritesSnapshot.length > 0 - ? liveWritesSnapshot - : undefined - : d.outputAssets - const script = - liveContentPath === p && d.script.content !== liveContentValue - ? { ...d.script, content: liveContentValue } - : d.script - if (script === d.script && outputAssets === d.outputAssets) { - return [p, d] as [string, Draft] - } - return [p, { ...d, script, outputAssets }] as [string, Draft] - }) - const activePath = activeDraftPath - const key = storageKey - const ws = $workspaceStore - const path = pipelineDraftPath - const hydrated = draftsHydrated - untrack(() => { - // Don't touch storage until the initial load settled. - if (!hydrated) return - const isEmpty = serialized.length === 0 && !activePath - const bundle: PipelineDraftBundle | undefined = isEmpty - ? undefined - : { drafts: serialized, activeDraftPath: activePath } - // localStorage crash mirror — synchronous, no debounce, no size cap. - try { - if (typeof localStorage !== 'undefined') { - if (isEmpty) localStorage.removeItem(key) - else - localStorage.setItem( - key, - encodeState({ drafts: serialized, activeDraftPath: activePath }) - ) - } - } catch (e) { - console.warn('failed to mirror pipeline state', e) - } - const serializedBundle = bundle ? JSON.stringify(bundle) : undefined - if (serializedBundle === lastPersistedBundle) return - lastPersistedBundle = serializedBundle - if (!ws) return - void UserDraftDbSyncer.save({ - workspace: ws, - itemKind: PIPELINE_DRAFT_KIND, - path, - // `null` deletes the bundle once the last draft is gone. - value: bundle ?? null, - auto: true - }) - }) - }) - - // Live-parsed annotations from whatever script is currently open in the - // details pane (draft or existing). Refreshed on every keystroke via - // `onAnnotationsChange`. Used to overlay unsaved schedule / trigger-asset - // edges onto the graph so the editor buffer and the graph stay in sync. - let liveAnnotations = $state<{ - scriptPath: string | undefined - annotations: PipelineAnnotations - }>({ - scriptPath: undefined, - annotations: { - inPipeline: false, - triggerAssets: [], - nativeTriggers: [], - dataTests: [], - columnLineage: [] - } - }) - - // Live-inferred body assets (read/write usages parsed by inferAssets - // — e.g. CREATE TABLE in SQL, loadS3File / writeS3File in TS/Python). - // Refreshed via onAssetsChange. We use the write subset as the - // authoritative output node set for drafts whose body has been edited - // past the seeded template; without this, renaming a CREATE TABLE - // target leaves the stale auto-output node on the graph. - let liveBodyAssets = $state<{ - scriptPath: string | undefined - assets: AssetWithAltAccessType[] - columnLineage?: ColumnLineage[] - }>({ scriptPath: undefined, assets: [] }) - - // The open draft's live editor buffer, emitted by the pane on every - // keystroke (`onContentChange`). The persist effect overlays it onto the - // Map's copy so autosave reflects in-progress edits — `onDraftPersist` - // only commits content into the Map on pane teardown, which would leave - // autosave a full editing session behind. - let liveContent = $state<{ scriptPath: string | undefined; content: string }>({ - scriptPath: undefined, - content: '' - }) - - // Canonical "empty" overlay literals, reused both as reset values for the - // live-* state above and as the no-overlay inputs to the deployed graph. + // The live editor overlays (annotations / body assets / content for the open + // script) now live in `pe`. The canonical "empty" literals stay here — they + // also seed the no-overlay inputs to the deployed graph below. const EMPTY_LIVE_ASSETS = { scriptPath: undefined, assets: [] } const EMPTY_LIVE_ANNOTATIONS = { scriptPath: undefined, @@ -518,16 +236,6 @@ } } - // Reset every live editor overlay (annotations / body assets / content) - // back to empty, unconditionally. Used by the leave-edit path so a stale - // buffer for the previously-open script can't leak into the view graphs. - // (forgetPath resets these per-path instead — see there.) - function clearLiveOverlays() { - liveAnnotations = EMPTY_LIVE_ANNOTATIONS - liveBodyAssets = EMPTY_LIVE_ASSETS - liveContent = { scriptPath: undefined, content: '' } - } - // Only-add cache of (script_path → body content) populated lazily by // `bodyFetchEffect` for every script in the current folder. We never // remove entries: stale keys (renamed-away, deleted) are simply ignored @@ -559,7 +267,9 @@ const g = graphRes.current if (!g) return { writes, reads } const liveAssetsForPath = (path: string) => - liveBodyAssets.scriptPath === path ? liveBodyAssets.assets : inferredAssetsByPath.get(path) + pe.liveBodyAssets.scriptPath === path + ? pe.liveBodyAssets.assets + : inferredAssetsByPath.get(path) for (const r of g.runnables) { if (r.usage_kind !== 'script') continue const assets = liveAssetsForPath(r.path) @@ -580,12 +290,12 @@ const out = new Map>() const g = graphRes.current if (!g) return out - const livePath = liveAnnotations.scriptPath + const livePath = pe.liveAnnotations.scriptPath for (const r of g.runnables) { if (r.usage_kind !== 'script') continue let kinds: Set if (r.path === livePath) { - kinds = new Set(liveAnnotations.annotations.nativeTriggers.map((n) => n.kind)) + kinds = new Set(pe.liveAnnotations.annotations.nativeTriggers.map((n) => n.kind)) } else { const body = bodiesByPath.get(r.path) if (!body) continue @@ -663,13 +373,17 @@ const script = buildDraft(language, scriptPath, triggers, outputKind, out, input) // Write the new draft into the map (structural update so Svelte // re-derives graphWithDraft) and focus it in the details pane. When - // the user picked `none`, `outputAsset` is undefined and the graph - // overlay skips synthesizing a write edge. - const next = new Map(drafts) - next.set(scriptPath, { localId: newDraftLocalId(), script, outputAsset: out }) - drafts = next - activeDraftPath = scriptPath - selection = undefined + // the user picked `none`, `out` is undefined and the graph overlay + // skips synthesizing a write edge. + const next = new Map(pe.drafts) + next.set(scriptPath, { + localId: pe.newDraftLocalId(), + script, + outputAssets: out ? [out] : undefined + }) + pe.drafts = next + pe.activeDraftPath = scriptPath + pe.selection = undefined // Follow the new node with a smooth pan. The id matches the runnable // node the canvas builds for a draft script (`script:`). @@ -713,6 +427,43 @@ aiChatManager.sendRequest({ instructions }) } + // ===================== AI chat pipeline integration ===================== + // The global AI chat (dev-gated) gains pipeline-building tools while this + // editor is mounted, via the helpers registered below. AI mutations don't + // deploy — they apply directly as unsaved drafts on the canvas (the same way + // the flow/script editor applies AI edits), which the user then deploys. The + // build/edit logic is shared verbatim with the in-session preview + // (PipelineEditorView) via createPipelineAiHelpers. + + const pipelineAiHelpers = createPipelineAiHelpers({ + getFolder: () => folder, + getWorkspace: () => $workspaceStore, + getResolvedGraph: () => graphWithDraft, + getDrafts: () => pe.drafts, + setDrafts: (next) => (pe.drafts = next), + newDraftLocalId: pe.newDraftLocalId, + onForgetPath: (path) => forgetPath(path), + onShowDrafts: () => (includeDrafts = true), + onProposeNode: (path) => focusPipelineNode(`script:${path}`), + ensureEditable: () => { + // Auto-enter edit so AI changes are visible/actionable, unless the user + // is an operator (no edit permission) — then refuse with a clear error. + if (isOperator) { + throw new Error('This pipeline is read-only for your role; AI edits are disabled.') + } + if (mode !== 'edit') setMode('edit') + }, + onRunStarted: (jobId, path) => { + activeRunnables.arm(`script:${path}`) + runsPendingJobId = jobId + runsRefreshKey++ + activeRunnable = { kind: 'script', path } + activeRunnableJobId = jobId + } + }) + + onMount(() => aiChatManager.setPipelineHelpers(pipelineAiHelpers)) + // Navigation guard state. `pendingNavigationUrl` holds the URL the user // tried to leave to so we can complete the navigation after they pick // "Save all" or "Discard all"; `bypassNavigationGuard` is the standard @@ -728,7 +479,7 @@ bypassNavigationGuard = false return } - if (drafts.size === 0) return + if (pe.drafts.size === 0) return // `leave` covers tab close / hard reload / cross-origin nav. SvelteKit // turns a cancelled leave into a browser-native "Leave site?" prompt, // which we explicitly don't want — match the rest of the editors and @@ -762,8 +513,8 @@ // Wipe every draft and the active selection so saved drafts and // stale active path don't bleed into the next page. localStorage // is overwritten by the persist effect on the next tick. - drafts = new Map() - activeDraftPath = undefined + pe.drafts = new Map() + pe.activeDraftPath = undefined saveErrors = new Map() const target = pendingNavigationUrl leaveModalOpen = false @@ -782,7 +533,7 @@ // page so they can deal with the failures via the bar's error // popover. Otherwise resume the navigation that triggered the // guard. - if (drafts.size === 0) { + if (pe.drafts.size === 0) { const target = pendingNavigationUrl leaveModalOpen = false pendingNavigationUrl = undefined @@ -871,10 +622,10 @@ } async function saveAllDrafts() { - if (!$workspaceStore || drafts.size === 0 || savingAll) return + if (!$workspaceStore || pe.drafts.size === 0 || savingAll) return savingAll = true const ws = $workspaceStore - const entries = [...drafts.entries()] + const entries = [...pe.drafts.entries()] // Snapshot what the preview promises for every draft before anything // deploys — used to verify the persisted graph below. const predicted = predictCascadeFacts(entries.map(([p]) => p)) @@ -905,22 +656,22 @@ // entries to keep insertion order stable. if (savedPaths.length > 0) { const next = new Map() - for (const [k, v] of drafts) { + for (const [k, v] of pe.drafts) { if (!savedPaths.includes(k)) next.set(k, v) } - drafts = next + pe.drafts = next // If the open draft just got deployed, transfer the focus to // its now-persisted runnable so the pane stays on the same // script the user was editing — otherwise the pane closes, // the canvas re-fits, and the user has to re-find their // script after every save. - if (activeDraftPath && savedPaths.includes(activeDraftPath)) { - selection = { + if (pe.activeDraftPath && savedPaths.includes(pe.activeDraftPath)) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', - path: activeDraftPath + path: pe.activeDraftPath } - activeDraftPath = undefined + pe.activeDraftPath = undefined } await graphRes.refetch() // Verify only what actually deployed — failed drafts would @@ -939,10 +690,10 @@ } function discardDraft(path: string) { - if (!drafts.has(path)) return - const next = new Map(drafts) + if (!pe.drafts.has(path)) return + const next = new Map(pe.drafts) next.delete(path) - drafts = next + pe.drafts = next forgetPath(path) } @@ -961,18 +712,18 @@ // selection + per-path save errors. `bodiesByPath` keeps its entry // (only-add cache, harmless if stale). function forgetPath(path: string) { - if (activeDraftPath === path) activeDraftPath = undefined - if (selection?.kind === 'runnable' && selection.path === path) { - selection = undefined + if (pe.activeDraftPath === path) pe.activeDraftPath = undefined + if (pe.selection?.kind === 'runnable' && pe.selection.path === path) { + pe.selection = undefined } - if (liveAnnotations.scriptPath === path) { - liveAnnotations = EMPTY_LIVE_ANNOTATIONS + if (pe.liveAnnotations.scriptPath === path) { + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS } - if (liveBodyAssets.scriptPath === path) { - liveBodyAssets = EMPTY_LIVE_ASSETS + if (pe.liveBodyAssets.scriptPath === path) { + pe.liveBodyAssets = EMPTY_LIVE_ASSETS } - if (liveContent.scriptPath === path) { - liveContent = { scriptPath: undefined, content: '' } + if (pe.liveContent.scriptPath === path) { + pe.liveContent = { scriptPath: undefined, content: '' } } clearSaveError(path) } @@ -983,13 +734,13 @@ // open and surface the conflict inline. function renameDraft(oldPath: string, newPath: string): boolean | string { if (oldPath === newPath) return true - const draft = drafts.get(oldPath) + const draft = pe.drafts.get(oldPath) if (!draft) return 'Draft not found' - if (drafts.has(newPath)) return 'Another draft already uses this path' + if (pe.drafts.has(newPath)) return 'Another draft already uses this path' const next = new Map() // Preserve insertion order: replace the entry at its original // position so the canvas / lists don't reshuffle on rename. - for (const [k, v] of drafts) { + for (const [k, v] of pe.drafts) { if (k === oldPath) { const updatedScript = { ...v.script, path: newPath } next.set(newPath, { ...v, script: updatedScript }) @@ -997,8 +748,8 @@ next.set(k, v) } } - drafts = next - if (activeDraftPath === oldPath) activeDraftPath = newPath + pe.drafts = next + if (pe.activeDraftPath === oldPath) pe.activeDraftPath = newPath // Path-keyed live overlays: re-key for the renamed draft so the // graph stays consistent between the moment we mutate `drafts` // here and the next editor event that re-emits annotations / @@ -1007,14 +758,14 @@ // re-applying live overlays against the same OLD path — leaving // phantom edges that displace the + node off the top of the // graph and shuffle the layout. - if (liveAnnotations.scriptPath === oldPath) { - liveAnnotations = { ...liveAnnotations, scriptPath: newPath } + if (pe.liveAnnotations.scriptPath === oldPath) { + pe.liveAnnotations = { ...pe.liveAnnotations, scriptPath: newPath } } - if (liveBodyAssets.scriptPath === oldPath) { - liveBodyAssets = { ...liveBodyAssets, scriptPath: newPath } + if (pe.liveBodyAssets.scriptPath === oldPath) { + pe.liveBodyAssets = { ...pe.liveBodyAssets, scriptPath: newPath } } - if (liveContent.scriptPath === oldPath) { - liveContent = { ...liveContent, scriptPath: newPath } + if (pe.liveContent.scriptPath === oldPath) { + pe.liveContent = { ...pe.liveContent, scriptPath: newPath } } // `inferredWritesByPath` / `inferredReadsByPath` / // `annotatedNativeKindsByPath` are derived from `g.runnables` × @@ -1055,7 +806,7 @@ >() function deployRenamedDraft(path: string) { - const draft = drafts.get(path) + const draft = pe.drafts.get(path) if (!draft) return const localId = draft.localId let state = deployQueue.get(localId) @@ -1078,7 +829,7 @@ try { while (true) { if (!$workspaceStore) break - const draft = drafts.get(path) + const draft = pe.drafts.get(path) if (!draft) break try { await saveDraft(path, draft, $workspaceStore) @@ -1111,12 +862,12 @@ // queued path is waiting, the next loop iteration will // pick it up and we keep the draft live. if (!state.queuedPath) { - const nextDrafts = new Map(drafts) + const nextDrafts = new Map(pe.drafts) nextDrafts.delete(path) - drafts = nextDrafts - if (activeDraftPath === path) { - selection = { kind: 'runnable', runnable_kind: 'script', path } - activeDraftPath = undefined + pe.drafts = nextDrafts + if (pe.activeDraftPath === path) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path } + pe.activeDraftPath = undefined } if (saveErrors.has(path)) { const nextErrors = new Map(saveErrors) @@ -1142,22 +893,22 @@ state.inflight = false // On the rare path where the draft is also gone (deployed + // no queued path), drop the slot to keep the map bounded. - if (!drafts.has(path) && !state.queuedPath) { + if (!pe.drafts.has(path) && !state.queuedPath) { deployQueue.delete(localId) } } } // Currently-open draft shape (if any) — fed into the details pane. - let activeDraft = $derived(activeDraftPath ? drafts.get(activeDraftPath) : undefined) + let activeDraft = $derived(pe.activeDraftPath ? pe.drafts.get(pe.activeDraftPath) : undefined) // Path of the script currently open in the details pane (draft or // persisted selection), used wherever run-routing / overlay logic needs // "the one script the user is editing right now". let openScriptPath = $derived( - activeDraftPath ?? - (selection?.kind === 'runnable' && selection.runnable_kind === 'script' - ? selection.path + pe.activeDraftPath ?? + (pe.selection?.kind === 'runnable' && pe.selection.runnable_kind === 'script' + ? pe.selection.path : undefined) ) @@ -1168,12 +919,12 @@ $effect(() => { if (mode !== 'edit') return if ( - selection?.kind === 'runnable' && - selection.runnable_kind === 'script' && - drafts.has(selection.path) + pe.selection?.kind === 'runnable' && + pe.selection.runnable_kind === 'script' && + pe.drafts.has(pe.selection.path) ) { - activeDraftPath = selection.path - selection = undefined + pe.activeDraftPath = pe.selection.path + pe.selection = undefined } }) // Symmetric demotion: view mode shows the deployed truth, so an open @@ -1185,95 +936,13 @@ if (mode !== 'view') return const d = activeDraft if (d && d.script.hash) { - selection = { kind: 'runnable', runnable_kind: 'script', path: d.script.path } - activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: d.script.path } + pe.activeDraftPath = undefined } }) - // Named handlers for the details pane's live callbacks. Inline arrows - // would be rebuilt on every parent re-render, and the pane's $effects - // track those refs as deps — combined with the drafts mutation in - // `handleDraftContentChange`, that creates a parent ↔ child feedback - // loop ("effect_update_depth_exceeded"). Named functions keep the - // prop reference stable so the $effects only re-fire on real - // content changes (e.g. handleContentChange below mutating drafts). - function handleAnnotationsChange( - scriptPath: string | undefined, - annotations: PipelineAnnotations - ) { - liveAnnotations = { scriptPath, annotations } - } - function handleAssetsChange( - scriptPath: string | undefined, - assets: AssetWithAltAccessType[], - columnLineage?: ColumnLineage[] - ) { - // Single update site for the live overlay. `inferredWritesByPath` - // / `inferredReadsByPath` are now derived from `liveBodyAssets` - // (for the open script) + `inferredAssetsByPath` (prefetched - // snapshot for every other script), so we don't have to write - // into those caches here — the derive picks up our update on the - // next reactive tick. - liveBodyAssets = { scriptPath, assets, columnLineage } - } - function handleContentChange(scriptPath: string | undefined, content: string) { - liveContent = { scriptPath, content } - } - function handleDraftPersist( - p: string, - snapshot: { - content: string - writes: { kind: AssetKind; path: string }[] - script?: Script - } - ) { - // Persist body edits + inferred outputs back into the drafts Map so - // they survive switching to another node and back (the details pane - // clones draftScript locally on every prop change, and `outputAsset` - // would otherwise stay frozen at the value seeded when the draft - // was opened — leaving a stale write edge on the canvas after the - // user has renamed a CREATE TABLE / writeS3File target). - // - // Deferred a microtask: this is called from the pane's $effect - // teardown, which observes the *previous* batch values — after a - // discard, `drafts` still appears to contain the discarded entry, - // and writing a map cloned from that stale read would resurrect it - // (the "discard needs two clicks" bug). One microtask later the - // batch has committed and the reads are fresh. - queueMicrotask(() => { - const d = drafts.get(p) - if (!d) { - // Unsaved edits to a *deployed* script (the pane only emits - // these when the buffer differs from the deployed content): - // promote to a draft so the work survives mode switches / - // selection changes, shows in the drafts chip, and deploys - // via Save all. The script snapshot carries the deployed - // hash, so saving chains a new version off it. - if (!snapshot.script) return - const next = new Map(drafts) - next.set(p, { - localId: newDraftLocalId(), - script: snapshot.script, - outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined - }) - drafts = next - return - } - const writesEqual = - d.outputAssets?.length === snapshot.writes.length && - (d.outputAssets ?? []).every( - (a, i) => a.kind === snapshot.writes[i]?.kind && a.path === snapshot.writes[i]?.path - ) - if (d.script.content === snapshot.content && writesEqual) return - const next = new Map(drafts) - next.set(p, { - ...d, - script: { ...d.script, content: snapshot.content }, - outputAssets: snapshot.writes.length > 0 ? snapshot.writes : undefined - }) - drafts = next - }) - } + // The details pane's live callbacks (annotations / assets / content / persist) + // now live on `pe` as stable arrow fields — pass `pe.handleX` straight through. // Canvas callbacks, named so the prop refs stay stable across re-renders // (same rationale as the live-callback handlers above) and so the @@ -1297,13 +966,13 @@ s && s.kind === 'runnable' && s.runnable_kind === 'script' && - drafts.has(s.path) + pe.drafts.has(s.path) ) { - activeDraftPath = s.path - selection = undefined + pe.activeDraftPath = s.path + pe.selection = undefined } else { - activeDraftPath = undefined - selection = s + pe.activeDraftPath = undefined + pe.selection = s } } function handleAddScriptForAsset( @@ -1349,8 +1018,8 @@ return } if (info.runnable_kind !== 'script') return - activeDraftPath = undefined - selection = { kind: 'runnable', runnable_kind: 'script', path: info.path } + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: info.path } requestRemoveSignal++ } async function handleRunProducer(producer: { @@ -1386,7 +1055,7 @@ if (cascade) { const hasDraftInChain = producer.unsaved === true || - computeDownstreamClosure(graphWithDraft, producer.path).nodes.some((p) => drafts.has(p)) + computeDownstreamClosure(graphWithDraft, producer.path).nodes.some((p) => pe.drafts.has(p)) if (hasDraftInChain) { return await runDraftAwareCascade(producer.path) } @@ -1403,7 +1072,7 @@ const skipArg = cascade ? {} : { _wmill_skip_asset_dispatch: true } let jobId: string | undefined if (producer.unsaved) { - const draft = drafts.get(producer.path) + const draft = pe.drafts.get(producer.path) if (!draft?.script.content || !draft.script.language) return undefined jobId = await JobService.runScriptPreview({ workspace: $workspaceStore, @@ -1483,9 +1152,9 @@ let graphWithDraft = $derived.by(() => resolveGraph({ base: graphRes.current ?? EMPTY_GRAPH, - drafts, - liveBodyAssets, - liveAnnotations, + drafts: pe.drafts, + liveBodyAssets: pe.liveBodyAssets, + liveAnnotations: pe.liveAnnotations, inferredWritesByPath, inferredReadsByPath, annotatedNativeKindsByPath @@ -1523,11 +1192,11 @@ // Guard before reassigning so an already-empty overlay doesn't // needlessly invalidate the graph derives every mode toggle. if ( - liveAnnotations.scriptPath != undefined || - liveBodyAssets.scriptPath != undefined || - liveContent.scriptPath != undefined + pe.liveAnnotations.scriptPath != undefined || + pe.liveBodyAssets.scriptPath != undefined || + pe.liveContent.scriptPath != undefined ) { - clearLiveOverlays() + pe.clearLiveOverlays() } }) }) @@ -1536,20 +1205,11 @@ // preserved, so toggling back to edit keeps them). Tracks activeDraftPath // too, covering the onMount localStorage restore landing after a view load. $effect(() => { - if (mode === 'view' && !includeDrafts && activeDraftPath != undefined) { - activeDraftPath = undefined + if (mode === 'view' && !includeDrafts && pe.activeDraftPath != undefined) { + pe.activeDraftPath = undefined } }) - // Selection highlights the active draft (if any) or the user's picked - // node. Non-active drafts render without selection highlight but are - // still clickable to re-enter their edit pane. - let effectiveSelection = $derived( - activeDraftPath - ? { kind: 'runnable', runnable_kind: 'script', path: activeDraftPath } - : selection - ) - // Bumped after every successful run dispatch so AssetRunsPanel re-fetches // the listing immediately — the new (preview or script) job appears in // the history popover without waiting on its 3 s poll tick. @@ -1696,7 +1356,7 @@ // drafts (same condition as `displayGraph`). Otherwise — View mode with // drafts hidden — a bounded run must execute the *deployed* scripts the // user is looking at, not preview jobs from hidden local drafts. - const draft = mode === 'edit' || includeDrafts ? drafts.get(path) : undefined + const draft = mode === 'edit' || includeDrafts ? pe.drafts.get(path) : undefined if (draft) { if (!draft.script.content || !draft.script.language) { throw new Error(`draft ${path} has no content/language`) @@ -1961,7 +1621,7 @@ // running a draft via runScriptPreview creates a `preview`-kind job at // the same path, which the panel's listing query picks up. let selectionProducers = $derived.by(() => { - const sel = selection + const sel = pe.selection if (!sel || sel.kind !== 'asset') return [] return graphWithDraft.edges .filter((e) => { @@ -1991,7 +1651,7 @@ // ducklake-asset selection so it isn't rebuilt on every editor keystroke when // the trace UI isn't even shown. let columnGraph = $derived( - selection?.kind === 'asset' && selection.asset_kind === 'ducklake' + pe.selection?.kind === 'asset' && pe.selection.asset_kind === 'ducklake' ? buildColumnGraph(displayGraph) : EMPTY_COLUMN_GRAPH ) @@ -2008,7 +1668,7 @@ // without it) is treated as unknown → evolvable, so captured history is never // hidden behind a stale "fixed" verdict. let schemaCanEvolve = $derived.by(() => { - const sel = selection + const sel = pe.selection if (!sel || sel.kind !== 'asset' || sel.asset_kind !== 'ducklake') return true const producerPaths = new Set(selectionProducers.map((p) => p.path)) const producers = graphWithDraft.runnables.filter((r) => producerPaths.has(r.path)) @@ -2068,7 +1728,7 @@ // flow. Drafts have no deployed endpoint yet, so nudge the user to save // first (mirrors openMissingTriggerDrawer). function openWebhookDrawer(scriptPath: string) { - if (drafts.has(scriptPath)) { + if (pe.drafts.has(scriptPath)) { sendUserToast( `Save the script "${scriptPath}" first — webhooks only trigger the deployed version.`, true @@ -2090,12 +1750,12 @@ // Same mode gate as handleCanvasSelect: view mode targets the // deployed script (its run form runs the deployed version), even // when unsaved edits were promoted to a draft. - if (mode === 'edit' && drafts.has(scriptPath)) { - activeDraftPath = scriptPath - selection = undefined + if (mode === 'edit' && pe.drafts.has(scriptPath)) { + pe.activeDraftPath = scriptPath + pe.selection = undefined } else { - activeDraftPath = undefined - selection = { kind: 'runnable', runnable_kind: 'script', path: scriptPath } + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: scriptPath } } // Bump after a tick so the pane has reacted to the new selection/draft // and begun mounting the run form before it hunts for the S3 input. @@ -2110,7 +1770,7 @@ // time or silently bind to nothing. Surface that as a toast and // keep the drawer closed; the user needs to save the script first // (which also creates it under the new path if they renamed it). - if (drafts.has(scriptPath)) { + if (pe.drafts.has(scriptPath)) { sendUserToast( `Save the script "${scriptPath}" first — triggers can only be attached to deployed scripts.`, true @@ -2239,7 +1899,7 @@ g.runnables .filter((r) => r.usage_kind === 'script') .map((r) => r.path) - .filter((p) => !drafts.has(p) && !bodiesByPath.has(p)) + .filter((p) => !pe.drafts.has(p) && !bodiesByPath.has(p)) ) if (targets.length === 0) return let i = 0 @@ -2351,8 +2011,8 @@ anchored between the two flex-1 side groups so it stays centered, with breathing room on both sides. -->
- setMode(m)} /> - {#if mode === 'view' && drafts.size > 0} + setMode(m)} /> + {#if mode === 'view' && pe.drafts.size > 0} {/if}
@@ -2410,7 +2070,7 @@ {/snippet} {/if} - {#if mode === 'edit' && drafts.size > 0} + {#if mode === 'edit' && pe.drafts.size > 0} @@ -2420,7 +2080,7 @@ itemKind="data_pipeline" path={pipelineDraftPath} draftOnly - loadedFromDraft={loadedFromDbDraft} + loadedFromDraft={pe.loadedFromDbDraft} /> {/if} {/if} {#if mode === 'view'} @@ -2470,257 +2130,152 @@ Failed to load pipeline: {graphRes.error.message} {:else} - - -
- - {#if boundPick} - -
- -
- - {boundPickEnds.size === 0 - ? 'Click end node(s) to bound the run' - : `${boundScripts.length} script${boundScripts.length === 1 ? '' : 's'} up to ${boundPickEnds.size} end${boundPickEnds.size === 1 ? '' : 's'}`} - - - from {boundPickStart ? shortPath(boundPickStart) : ''} - -
- - + (panelHidden = !panelHidden)} + {prefetchingAssets} + hoveredPaths={activityHoverPaths} + selectedRunPaths={activitySelectPaths} + {activeRunnable} + activeRunnableIds={activeRunnables.ids} + runStates={mergedRunStates} + eventLogEvents={activeRunnables.events} + {runsRefreshKey} + {runsPendingJobId} + {boundPick} + validStartPaths={isOperator ? undefined : validStartPaths} + onStartBoundedRun={isOperator ? undefined : startBoundedRun} + onPickEnd={pickBoundEnd} + {panToNodeId} + onCreateMissingTrigger={mode === 'edit' ? openMissingTriggerDrawer : undefined} + onEditTrigger={mode === 'edit' ? openEditTriggerDrawer : undefined} + onDeleteTrigger={mode === 'edit' ? deleteAttachedTrigger : undefined} + onOpenWebhook={openWebhookDrawer} + onOpenDataUpload={openDataUploadRun} + onSelect={handleCanvasSelect} + onAddScriptForAsset={mode === 'edit' ? handleAddScriptForAsset : undefined} + onAddPipelineScript={mode === 'edit' ? handleAddPipelineScript : undefined} + onRunnableMenuRemove={mode === 'edit' ? handleRunnableMenuRemove : undefined} + onRunProducer={mode === 'edit' ? handleRunProducer : undefined} + onRequestEdit={isOperator ? undefined : () => setMode('edit')} + canRunByPath={openScriptHasDataUpload} + onRunByPath={runByPathLegit} + {selectionProducers} + selectionColumnGraph={pe.activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} + {schemaCanEvolve} + downstreamSubscribers={editedScriptDownstreamCount} + onStartBoundedRunForOpen={startBoundedRun} + canBoundedRunOpenScript={!!openScriptPath && + validStartPaths.has(openScriptPath) && + lineageDownstreamPaths.has(openScriptPath)} + onRunCompleted={() => { + activeRunnable = undefined + activeRunnableJobId = undefined + }} + onTestStateChange={(running) => { + const openPath = openScriptPath + if (running && openPath) { + activeRunnable = { kind: 'script', path: openPath } + activeRunnables.arm(`script:${openPath}`) + activeRunnableJobId = undefined + } else if (!running && activeRunnable?.path === openPath) { + activeRunnable = undefined + activeRunnableJobId = undefined + } + }} + {requestRemoveSignal} + {requestRunSignal} + {requestRunCascadeSignal} + focusUploadSignal={focusDataUploadSignal} + onDraftPathChange={renameDraft} + onClose={() => { + pe.selection = undefined + pe.activeDraftPath = undefined + pe.liveAnnotations = EMPTY_LIVE_ANNOTATIONS + }} + onDiscard={() => { + if (pe.activeDraftPath) discardDraft(pe.activeDraftPath) + }} + onDraftSaved={async (savedPath) => { + const predicted = predictCascadeFacts([savedPath]) + const nextDrafts = new Map(pe.drafts) + nextDrafts.delete(savedPath) + pe.drafts = nextDrafts + if (pe.activeDraftPath === savedPath) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: savedPath } + pe.activeDraftPath = undefined + } + clearSaveError(savedPath) + await graphRes.refetch() + reportDeployDrift(predicted) + }} + onPersistedSaved={async (savedPath) => { + const predicted = predictCascadeFacts([savedPath]) + await graphRes.refetch() + reportDeployDrift(predicted) + }} + onScriptRenamed={async (oldPath, newPath) => { + if (pe.selection?.kind === 'runnable' && pe.selection.path === oldPath) { + pe.selection = { ...pe.selection, path: newPath } + } + await graphRes.refetch() + }} + onScriptRemoved={async (removedPath) => { + forgetPath(removedPath) + await graphRes.refetch() + }} + > + {#snippet boundBar()} + {#if boundPick} +
+ +
+ + {boundPickEnds.size === 0 + ? 'Click end node(s) to bound the run' + : `${boundScripts.length} script${boundScripts.length === 1 ? '' : 's'} up to ${boundPickEnds.size} end${boundPickEnds.size === 1 ? '' : 's'}`} + + + from {boundPickStart ? shortPath(boundPickStart) : ''} +
- {/if} - {#if mode === 'edit'} - - - {/if} - {#if prefetchingAssets} -
Cancel +
- {/if} - {#if mode !== 'edit' || selection != undefined || activeDraftPath != undefined} - -
-
- {/if} -
- {#if detailsPaneOpen && $workspaceStore} - - {#if mode !== 'edit' && selection == undefined && activeDraftPath == undefined} - - (activityHoverPaths = p ?? [])} - onSelectRun={(p) => (activitySelectPaths = p ?? [])} - /> - {:else} - setMode('edit')} - canRunByPath={openScriptHasDataUpload} - onRunByPath={runByPathLegit} - selection={activeDraft ? undefined : selection} - selectionProducers={activeDraft ? [] : selectionProducers} - selectionColumnGraph={activeDraft ? EMPTY_COLUMN_GRAPH : columnGraph} - {schemaCanEvolve} - {runsRefreshKey} - {runsPendingJobId} - {activeRunnable} - downstreamSubscribers={editedScriptDownstreamCount} - onStartBoundedRun={openScriptPath && - validStartPaths.has(openScriptPath) && - lineageDownstreamPaths.has(openScriptPath) - ? () => startBoundedRun(openScriptPath!) - : undefined} - onRunCompleted={() => { - activeRunnable = undefined - activeRunnableJobId = undefined - }} - onTestStateChange={(running) => { - // Bridge: ScriptEditor's Test button triggers the - // same canvas-level "is running" hint as the - // per-node Run button. The currently-edited script - // is whichever path is open in the pane (active - // draft, or the persisted-script selection). - const openPath = openScriptPath - if (running && openPath) { - activeRunnable = { kind: 'script', path: openPath } - // Mark the tested runnable as launched-from-here so the - // folder poll's catch-up pulse won't re-flash its edge a - // poll-interval after a fast job already finished (the - // edge is animated zero-latency by `activeRunnable`, and - // the test loader clears that the instant it completes). - // Also upgrades to the fast poll so the badge lands sooner. - activeRunnables.arm(`script:${openPath}`) - // Editor Test path clears via its own callbacks, not - // the job-id effect — drop any stale tracked id so a - // prior canvas run's completion can't clear this hint. - activeRunnableJobId = undefined - } else if (!running && activeRunnable?.path === openPath) { - activeRunnable = undefined - activeRunnableJobId = undefined - } - }} - {requestRemoveSignal} - {requestRunSignal} - {requestRunCascadeSignal} - focusUploadSignal={focusDataUploadSignal} - draftScript={activeDraft?.script} - {pathPrefix} - onDraftPathChange={renameDraft} - workspace={$workspaceStore} - onAnnotationsChange={handleAnnotationsChange} - onAssetsChange={handleAssetsChange} - onContentChange={handleContentChange} - onDraftPersist={handleDraftPersist} - onclose={() => { - // Close dismisses the pane but preserves drafts so - // the user can come back to them. Discarding is - // via the explicit "Discard" button in the pane. - selection = undefined - activeDraftPath = undefined - liveAnnotations = EMPTY_LIVE_ANNOTATIONS - }} - onHide={() => (panelHidden = true)} - onDiscard={() => { - if (activeDraftPath) discardDraft(activeDraftPath) - }} - onDraftSaved={async (savedPath) => { - // Snapshot the preview's promise while the draft - // overlay still exists (dropped from `drafts` below). - const predicted = predictCascadeFacts([savedPath]) - // Drop the now-deployed draft and hand focus to its - // persisted runnable so the pane stays open on the - // same script. `discardDraft` would clear - // activeDraftPath without setting selection — the - // canvas would deselect and the view reset on the - // next refetch. - const nextDrafts = new Map(drafts) - nextDrafts.delete(savedPath) - drafts = nextDrafts - if (activeDraftPath === savedPath) { - selection = { - kind: 'runnable', - runnable_kind: 'script', - path: savedPath - } - activeDraftPath = undefined - } - clearSaveError(savedPath) - await graphRes.refetch() - reportDeployDrift(predicted) - }} - onPersistedSaved={async (savedPath) => { - // Snapshot before the refetch replaces the base graph - // — the live editor overlay is the prediction here. - const predicted = predictCascadeFacts([savedPath]) - // Refresh the asset graph so the rows the deploy - // just inserted (from the body-asset write list we - // pass at save time) make it into base.edges. The - // in-memory `inferredWritesByPath` overlay - // dedupes against base, so the edge stays put - // instead of flickering when the ScriptEditor - // remounts on the new hash. - await graphRes.refetch() - reportDeployDrift(predicted) - }} - onScriptRenamed={async (oldPath, newPath) => { - // Repoint the selection at the new path before the - // graph refetches so the pane stays focused on the - // same script. Order matters: update selection - // first, then refetch — otherwise the resource - // driving the pane would briefly resolve to nothing. - if (selection?.kind === 'runnable' && selection.path === oldPath) { - selection = { ...selection, path: newPath } - } - await graphRes.refetch() - }} - onScriptRemoved={async (removedPath) => { - // Drop every path-keyed overlay / cache entry - // pointing at the now-archived runnable so - // resolveGraph doesn't keep emitting lineage - // edges or missing-trigger placeholders against - // a script that no longer exists. Without this - // the inferred writes / annotation maps would - // keep dragging phantom nodes onto the canvas - // until the next folder change. - forgetPath(removedPath) - await graphRes.refetch() - }} - /> - {/if} - - {/if} - + Run selection + +
+ {/if} + {/snippet} + {#snippet idlePane()} + (activityHoverPaths = p ?? [])} + onSelectRun={(p) => (activitySelectPaths = p ?? [])} + /> + {/snippet} + {/if}
@@ -2761,18 +2316,18 @@

- {drafts.size === 1 ? 'Unsaved draft' : `${drafts.size} unsaved drafts`} + {pe.drafts.size === 1 ? 'Unsaved draft' : `${pe.drafts.size} unsaved drafts`}

- You have {drafts.size === 1 + You have {pe.drafts.size === 1 ? 'a draft pipeline script' - : `${drafts.size} draft pipeline scripts`} that {drafts.size === 1 + : `${pe.drafts.size} draft pipeline scripts`} that {pe.drafts.size === 1 ? 'has' : 'have'} not been deployed yet. What would you like to do?

    - {#each [...drafts.keys()] as p} + {#each [...pe.drafts.keys()] as p}
  • {p}
    • {/each}
@@ -2787,7 +2342,9 @@ unifiedSize="sm" startIcon={{ icon: leaveSaving ? Loader2 : Save }} > - {leaveSaving ? 'Saving…' : `Save all (${drafts.size})`} + {leaveSaving ? 'Saving…' : `Save all (${pe.drafts.size})`}
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. + +\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- \`// materialize append\` → INSERT-append rows (incremental); +- \`// materialize key=\` → merge/upsert on \`\`. + +\`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. + +\`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. +2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`), language chosen for the work: \`duckdb\` or \`postgresql\` for SQL-shaped data work, \`bun\`/\`python3\` for general transforms. SQL-heavy lakehouse steps usually use \`duckdb\`. +3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the \`/pipeline/\` editor open, prefer the dedicated \`build_pipeline_node\` / \`edit_pipeline_node\` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake table): + +\`\`\`sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +\`\`\` + +Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): + +\`\`\`sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +\`\`\` +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md new file mode 100644 index 0000000000..a0f2e2492a --- /dev/null +++ b/system_prompts/base/pipeline-base.md @@ -0,0 +1,60 @@ +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`. + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. +- **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. + +## Materialize (the managed output) + +> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. + +`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- `// materialize append` → INSERT-append rows (incremental); +- `// materialize key=` → merge/upsert on ``. + +`// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time. + +`materialize` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: `f//`. The folder is the pipeline. +2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`. +3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table): + +```sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +``` + +Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): + +```sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 6ee1971f43..98c03c4367 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -2369,6 +2369,7 @@ def main(): flow_base = read_markdown_file(base_dir / "flow-base.md") resources_base = read_markdown_file(base_dir / "resources.md") raw_app_base = read_markdown_file(base_dir / "raw-app.md") + pipeline_base = read_markdown_file(base_dir / "pipeline-base.md") workflow_as_code_base = read_markdown_file(base_dir / "workflow-as-code.md") flow_cli = read_markdown_file(base_dir / "flow-cli.md") flow_chat_special_modules = read_markdown_file(base_dir / "flow-chat-special-modules.md") @@ -2435,6 +2436,7 @@ def main(): 'FLOW_BASE': flow_base, 'RESOURCES_BASE': resources_base, 'RAW_APP_BASE': raw_app_base, + 'PIPELINE_BASE': pipeline_base, 'WORKFLOW_AS_CODE_BASE': workflow_as_code_base, 'FLOW_CHAT_SPECIAL_MODULES': flow_chat_special_modules, @@ -2534,6 +2536,11 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } +// Helper for data pipeline authoring (chat consumers) +export function getPipelinePrompt(): string { + return prompts.PIPELINE_BASE; +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2586,6 +2593,7 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; +export declare function getPipelinePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ From 9b65161c643bf3f120d2ebd82f786c17233a971b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 29 Jun 2026 22:37:39 +0200 Subject: [PATCH 03/56] fix(gcp): require token verification for authenticated push delivery (#9834) * fix(gcp): require token verification for authenticated push delivery Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 This commit updates the EE repository reference after PR #636 was merged in windmill-ee-private. Previous ee-repo-ref: 8c63d487c486002baf09c77ab937fd77a91765eb New ee-repo-ref: 38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ce3db8ebcb..4dd9981afc 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -95352c13c4c82247d8cfd80936f9203aeb079802 +38e87caeca6a1dce9e4f3fa029ac36dffb30f1b2 From a9ffdb996b418ad51a84973ff01dcd3f2c820f16 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 29 Jun 2026 22:41:38 +0200 Subject: [PATCH 04/56] chore(main): release 1.743.0 (#9837) * chore(main): release 1.743.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 13 ++ backend/Cargo.lock | 172 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 139 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b6fbd91..77316bdff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29) + + +### Features + +* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005)) +* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0)) + + +### Bug Fixes + +* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b)) + ## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7072389c21..c154084c51 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -2601,9 +2601,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.89+curl-8.20.0" +version = "0.4.90+curl-8.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" +checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" dependencies = [ "cc", "libc", @@ -6085,9 +6085,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -12922,9 +12922,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -13734,7 +13734,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13816,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.742.0" +version = "1.743.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13849,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13862,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14000,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14023,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14038,7 +14038,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14064,7 +14064,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.742.0" +version = "1.743.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14074,7 +14074,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14091,7 +14091,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14113,7 +14113,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14136,7 +14136,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14152,7 +14152,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14173,7 +14173,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14194,7 +14194,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14208,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -14243,7 +14243,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14268,7 +14268,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14286,7 +14286,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14308,7 +14308,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14328,7 +14328,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14365,7 +14365,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14393,7 +14393,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.742.0" +version = "1.743.0" dependencies = [ "lazy_static", "serde", @@ -14405,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.742.0" +version = "1.743.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14430,7 +14430,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14444,7 +14444,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.742.0" +version = "1.743.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14477,7 +14477,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.742.0" +version = "1.743.0" dependencies = [ "chrono", "lazy_static", @@ -14491,7 +14491,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14510,7 +14510,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.743.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14612,7 +14612,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.742.0" +version = "1.743.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14631,7 +14631,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.742.0" +version = "1.743.0" dependencies = [ "regex", "serde", @@ -14646,7 +14646,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14670,7 +14670,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "futures", @@ -14687,7 +14687,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.743.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14703,7 +14703,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -14724,7 +14724,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -14755,7 +14755,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "arc-swap", @@ -14780,7 +14780,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-stream", @@ -14814,7 +14814,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "futures", @@ -14832,7 +14832,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.743.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14841,7 +14841,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14853,7 +14853,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14865,7 +14865,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "gosyn", @@ -14877,7 +14877,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14889,7 +14889,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14901,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "nu-parser", @@ -14912,7 +14912,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14923,7 +14923,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14935,7 +14935,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14946,7 +14946,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -14968,7 +14968,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -14980,7 +14980,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +14994,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15011,7 +15011,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -15024,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -15036,7 +15036,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -15054,7 +15054,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15070,7 +15070,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15086,7 +15086,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -15097,7 +15097,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -15136,7 +15136,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "const_format", @@ -15175,7 +15175,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.742.0" +version = "1.743.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15186,7 +15186,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15244,7 +15244,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15277,7 +15277,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15310,7 +15310,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15330,7 +15330,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15364,7 +15364,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15400,7 +15400,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15423,7 +15423,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-nats", @@ -15471,7 +15471,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15506,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15534,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-trait", @@ -15559,7 +15559,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15578,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-once-cell", @@ -15688,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.742.0" +version = "1.743.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 45682215cf..725f460df1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.742.0" +version = "1.743.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.742.0" +version = "1.743.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 096d93163b..31e3ee26b6 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.743.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.743.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.743.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.743.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2afd2483b..62faf03ec3 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.742.0" +version = "1.743.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f9a15ef72d..18b1914826 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.742.0 + version: 1.743.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f8cc748bdb..38995ac0e9 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.742.0"; +export const VERSION = "v1.743.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 42d8109c49..7be2d4df35 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.742.0"; +export const VERSION = "1.743.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4a3fcf0696..0d66dfed67 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index e9ec640b02..0c122ffac7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.742.0", + "version": "1.743.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 7f9d00903a..1dc2caa039 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.742.0" +wmill = ">=1.743.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index f4342930e9..b2b394c836 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.742.0 + version: 1.743.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index e5b553e6f3..500f453c30 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.742.0' + ModuleVersion = '1.743.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 5310b72896..508f2376c1 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.742.0" +version = "1.743.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7ae2023a7c..650ccf5be0 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.742.0", + "version": "1.743.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8edc248c45..0a22c432d7 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.742.0", + "version": "1.743.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index bee795f5f0..9926885433 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.742.0 +1.743.0 From a27e814a03c615259381eaf684aa90d56569b0af Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 30 Jun 2026 10:10:31 +0200 Subject: [PATCH 05/56] feat: add copy-to-clipboard button to rendered Mermaid diagrams in AI chat (#9838) MermaidDisplay only showed the rendered SVG, hiding the raw source once rendering succeeded. Add a copy button in the showSvg branch mirroring the pattern in HighlightCode.svelte so the diagram source can be extracted. Fixes WIN-2109 Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/script/MermaidDisplay.svelte | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte index 06bcb63d98..4adf86e76d 100644 --- a/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/script/MermaidDisplay.svelte @@ -1,6 +1,9 @@ {#if showSvg} -
- - {@html svg} +
+
{:else} From a37a144e81cf6b3de935688a617e9d0e1756004a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 30 Jun 2026 17:49:57 +0200 Subject: [PATCH 06/56] fix(ai-chat): replay anthropic turns verbatim to keep thinking valid (#9843) Co-authored-by: Claude Opus 4.8 (1M context) --- .../components/copilot/chat/anthropic.test.ts | 164 ++++++++++++++++++ .../lib/components/copilot/chat/anthropic.ts | 94 +++++++--- 2 files changed, 230 insertions(+), 28 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/anthropic.test.ts diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts new file mode 100644 index 0000000000..9c9a4d0bb0 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it, vi } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { convertOpenAIToAnthropicMessages } from './anthropic' + +// anthropic.ts pulls in the chat client/registry layer at import time; the +// converter under test is pure, so stub those side-effecting modules away. +vi.mock('../lib', () => ({ + getProviderAndCompletionConfig: vi.fn(), + workspaceAIClients: {} +})) + +vi.mock('../reasoningRegistry', () => ({ + applyReasoningToConfig: vi.fn() +})) + +vi.mock('./shared', () => ({ + processToolCall: vi.fn() +})) + +describe('convertOpenAIToAnthropicMessages', () => { + it('replays a captured assistant turn verbatim, skips the standalone text copy, and leaves the turn untouched', () => { + const anthropicContent = [ + { type: 'thinking', thinking: 'first', signature: 'sig-1' }, + { + type: 'server_tool_use', + id: 'srv_1', + name: 'web_search', + input: { query: 'nist password length' } + }, + { + type: 'web_search_tool_result', + tool_use_id: 'srv_1', + content: [{ type: 'web_search_result', title: 'NIST', url: 'https://nist.gov' }] + }, + { type: 'thinking', thinking: 'second', signature: 'sig-2' }, + { type: 'tool_use', id: 'tool_1', name: 'list_resources', input: {} } + ] + // Snapshot to assert the stored content is never mutated by the converter. + const anthropicContentSnapshot = JSON.parse(JSON.stringify(anthropicContent)) + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'find the nist length then list resources' }, + // Standalone text the streamer emits before the tool-call message. + { role: 'assistant', content: 'Let me search the web.' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_1', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicContent: anthropicContent + } as any, + { role: 'tool', tool_call_id: 'tool_1', content: 'resource A, resource B' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + // user + the verbatim assistant turn + the tool result; the standalone text is dropped. + expect(out).toHaveLength(3) + expect(out[0]).toEqual({ role: 'user', content: 'find the nist length then list resources' }) + + // Assistant turn replayed in original order, no reordering or dropped blocks. + expect(out[1].role).toBe('assistant') + expect((out[1].content as any[]).map((b) => b.type)).toEqual([ + 'thinking', + 'server_tool_use', + 'web_search_tool_result', + 'thinking', + 'tool_use' + ]) + // The verbatim turn must stay byte-identical — no cache_control injected into it, + // or a thinking-block signature would no longer validate. + expect(out[1].content).toEqual(anthropicContentSnapshot) + expect((out[1].content as any[]).some((b) => 'cache_control' in b)).toBe(false) + expect(anthropicContent).toEqual(anthropicContentSnapshot) + + // The cache breakpoint lands on the trailing tool result, not the assistant turn. + expect(out[2]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'tool_1', + content: 'resource A, resource B', + cache_control: { type: 'ephemeral' } + } + ] + }) + }) + + it('converts a plain text assistant turn (no tools) and caches the trailing text block', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'hello' }, + { role: 'assistant', content: 'hi there' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(2) + expect(out[0]).toEqual({ role: 'user', content: 'hello' }) + expect(out[1].role).toBe('assistant') + expect(out[1].content).toEqual([ + { type: 'text', text: 'hi there', cache_control: { type: 'ephemeral' } } + ]) + }) + + it('falls back to _anthropicThinkingBlocks for turns persisted before _anthropicContent', () => { + const thinkingBlocks = [{ type: 'thinking', thinking: 'reasoning', signature: 'sig-old' }] + + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'do a thing' }, + // The standalone text persisted alongside an old-style turn must NOT be skipped + // (the fallback reconstruction relies on it for the assistant text). + { role: 'assistant', content: 'working on it' }, + { + role: 'assistant', + tool_calls: [ + { + id: 'tool_old', + type: 'function', + function: { name: 'list_resources', arguments: '{}' } + } + ], + _anthropicThinkingBlocks: thinkingBlocks + } as any + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(3) + expect(out[1]).toEqual({ role: 'assistant', content: 'working on it' }) + const content = out[2].content as any[] + // Thinking block re-injected first, then the tool_use. + expect(content.map((b) => b.type)).toEqual(['thinking', 'tool_use']) + expect(content[0]).toEqual(thinkingBlocks[0]) + expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' }) + }) + + it('caches a trailing tool result even when the prior turn used no captured content', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'q' }, + { + role: 'assistant', + tool_calls: [ + { id: 't1', type: 'function', function: { name: 'list_resources', arguments: '{}' } } + ] + } as any, + { role: 'tool', tool_call_id: 't1', content: 'done' } + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + const last = out[out.length - 1] + expect(last.role).toBe('user') + expect((last.content as any[])[0]).toMatchObject({ + type: 'tool_result', + tool_use_id: 't1', + cache_control: { type: 'ephemeral' } + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 48b068039f..b6e7e57a17 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -286,15 +286,15 @@ export async function parseAnthropicCompletion( role: 'assistant', tool_calls: toolCallsToProcess } - // Preserve thinking blocks (with signatures) so the next request keeps the - // reasoning chain — Anthropic requires this when thinking is combined with tool - // use. They are re-injected by convertOpenAIToAnthropicMessages. - const thinkingBlocks = finalMessage.content.filter( - (b) => b.type === 'thinking' || b.type === 'redacted_thinking' - ) - if (thinkingBlocks.length > 0) { - ;(assistantWithTools as any)._anthropicThinkingBlocks = thinkingBlocks - } + // Preserve the assistant turn verbatim (thinking/redacted_thinking with their + // signatures, server_tool_use + web_search_tool_result, text and tool_use) in + // original order. Anthropic binds each thinking block's signature to the blocks + // that precede it in the latest assistant message, so when this turn is replayed + // to continue past its tool call it must be byte-identical: reordering thinking to + // the front or dropping the web-search blocks invalidates a later block's + // signature and the request 400s with "thinking blocks ... cannot be modified". + // convertOpenAIToAnthropicMessages replays this content as-is. + ;(assistantWithTools as any)._anthropicContent = finalMessage.content messages.push(assistantWithTools) addedMessages.push(assistantWithTools) @@ -323,7 +323,33 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage let system: TextBlockParam[] | undefined const anthropicMessages: MessageParam[] = [] - for (const message of messages) { + // A streamed assistant turn that ends in tool calls is persisted as one or more + // standalone text messages followed by the tool-call message carrying + // _anthropicContent. That text is already inside _anthropicContent (replayed verbatim + // below), so drop the standalone copies — otherwise the text is duplicated and + // emitted ahead of the turn's thinking blocks. The scan stops at the preceding + // user/tool message, so only the current turn's own text is skipped. + const skipStandaloneText = new Set() + for (let i = 0; i < messages.length; i++) { + if (!(messages[i] as any)._anthropicContent) continue + for (let j = i - 1; j >= 0; j--) { + const m = messages[j] + if ( + m.role === 'assistant' && + typeof m.content === 'string' && + !m.tool_calls && + !(m as any)._anthropicContent + ) { + skipStandaloneText.add(j) + } else { + break + } + } + } + + for (let i = 0; i < messages.length; i++) { + const message = messages[i] + if (skipStandaloneText.has(i)) continue if (message.role === 'system') { const systemText = typeof message.content === 'string' ? message.content : JSON.stringify(message.content) @@ -345,10 +371,19 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage typeof message.content === 'string' ? message.content : JSON.stringify(message.content) }) } else if (message.role === 'assistant') { + // Replay a captured assistant turn verbatim so its thinking-block signatures + // stay valid (see the _anthropicContent note where the streamed turn is stored). + const anthropicContent = (message as any)._anthropicContent + if (Array.isArray(anthropicContent) && anthropicContent.length > 0) { + anthropicMessages.push({ role: 'assistant', content: anthropicContent as any }) + continue + } + const content: any[] = [] - // Re-inject preserved thinking blocks first (Anthropic requires thinking to - // precede tool_use in the same assistant turn when thinking is enabled). + // Fallback for sessions persisted before _anthropicContent existed: re-inject + // the preserved thinking blocks first (Anthropic requires thinking to precede + // tool_use in the same assistant turn when thinking is enabled). const thinkingBlocks = (message as any)._anthropicThinkingBlocks if (Array.isArray(thinkingBlocks) && thinkingBlocks.length > 0) { content.push(...thinkingBlocks) @@ -404,26 +439,29 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage } } - // Add cache_control to the last message content blocks + // Cache the conversation prefix: put an ephemeral breakpoint on the last content + // block of the last message. Each continuation only appends a tool result plus the + // next turn, so everything up to here is read from cache — which is what keeps + // replaying assistant turns verbatim (web-search results included) affordable. + // cache_control is valid on text/tool_use/tool_result blocks, but a thinking or + // redacted_thinking block must never be modified, so skip the breakpoint there. if (anthropicMessages.length > 0) { const lastMessage = anthropicMessages[anthropicMessages.length - 1] - if (Array.isArray(lastMessage.content)) { - // Add cache_control to the last content block - if (lastMessage.content.length > 0) { - const lastBlock = lastMessage.content[lastMessage.content.length - 1] - if (lastBlock.type === 'text') { - lastBlock.cache_control = { type: 'ephemeral' } - } - } - } else if (typeof lastMessage.content === 'string') { - // Convert string content to array format with cache_control + if (typeof lastMessage.content === 'string') { lastMessage.content = [ - { - type: 'text', - text: lastMessage.content, - cache_control: { type: 'ephemeral' } - } + { type: 'text', text: lastMessage.content, cache_control: { type: 'ephemeral' } } ] + } else if (Array.isArray(lastMessage.content) && lastMessage.content.length > 0) { + const lastIndex = lastMessage.content.length - 1 + const lastBlock = lastMessage.content[lastIndex] as any + if (lastBlock.type !== 'thinking' && lastBlock.type !== 'redacted_thinking') { + // Clone the block instead of mutating in place: the array may be a verbatim + // _anthropicContent turn that must stay unaltered for later requests. + lastMessage.content = [ + ...lastMessage.content.slice(0, lastIndex), + { ...lastBlock, cache_control: { type: 'ephemeral' } } + ] + } } } From 83ed011e264f20ffa66a7bf933f2fe3615cf6b67 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 30 Jun 2026 17:57:26 +0200 Subject: [PATCH 07/56] feat(object-store): make GCS service account key optional for Workload Identity (#9842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_gcs_client always called `.with_service_account_key(...)`, so an absent key (the settings UI stores "no key" as the empty JSON object `{}`) was handed to the builder and failed to parse instead of falling through to the object_store crate's InstanceCredentialProvider. Skip the call when the key is blank so GCS uses the instance's ambient credentials (GKE Workload Identity / the GCP metadata server). "Blank" (empty/whitespace/`{}`/`null`) is centralized in a shared `gcs_service_account_key_is_blank` predicate so the build path and the non-super-admin connectivity-test SSRF guard (`validate_object_storage_test`) agree on what counts as "no key" — otherwise a blank key would bypass the guard yet still trigger the ambient-credential fallback, letting an untrusted caller probe arbitrary buckets with the server's instance role. Also clarify the settings UI hint that the key may be left empty for ambient credentials, and add regression tests for the blank-key build path and the guard. Fixes WIN-2110 Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api-settings/src/lib.rs | 26 ++++++++- backend/windmill-object-store/src/lib.rs | 58 ++++++++++++++++++- .../ObjectStoreConfigSettings.svelte | 5 +- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a7e1322ad9..8339df7b76 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -406,7 +406,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul ) } ObjectSettings::Gcs(gcs) => { - if gcs.service_account_key.is_empty() { + // Mirror `build_gcs_client`'s blank-key check (shared predicate): a blank/`{}` key falls + // back to the instance's ambient credentials there, so it must be rejected here too — + // otherwise an untrusted caller could probe with the server's identity (the very + // SSRF/credential-exfil this function guards against). + if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { return Err(error::Error::NotAuthorized( "Testing GCS storage without a service account key requires a super admin" .to_string(), @@ -2187,6 +2191,26 @@ mod object_storage_test_hardening { ); } + #[tokio::test] + async fn rejects_gcs_blank_service_account_key() { + // A blank key makes build_gcs_client fall back to the instance's ambient credentials, so an + // untrusted caller must not be allowed to test with it. The `serviceAccountKey` field is + // serialized via serde's `as_string` (`to_string` of the JSON value), so the settings UI's + // "no key" empty object arrives as `"{}"` and a null as `"null"` — both must be rejected. + for key in [serde_json::json!({}), serde_json::json!(null)] { + let settings: ObjectSettings = serde_json::from_value(serde_json::json!({ + "type": "Gcs", + "bucket": "b", + "serviceAccountKey": key + })) + .unwrap(); + assert!( + validate_object_storage_test(&settings).await.is_err(), + "blank key {key:?} should be rejected" + ); + } + } + fn ip(s: &str) -> IpAddr { s.parse().unwrap() } diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cf30efe116..553eb69917 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -494,6 +494,23 @@ fn build_azure_blob_client( return Ok(Arc::new(store)); } +/// Whether a GCS `service_account_key` carries no static credentials, in which case the client +/// should fall back to the instance's ambient credentials (GKE Workload Identity / metadata server) +/// instead of being handed an unparseable key. Besides an empty/whitespace string, the settings UI +/// stores "no key" as an empty JSON object `{}` (and `serde_json` may yield `null`), so treat those +/// as absent too. Shared with the connectivity-test SSRF guard so both agree on what "no key" means. +pub fn gcs_service_account_key_is_blank(service_account_key: &str) -> bool { + let trimmed = service_account_key.trim(); + if trimmed.is_empty() { + return true; + } + match serde_json::from_str::(trimmed) { + Ok(serde_json::Value::Null) => true, + Ok(serde_json::Value::Object(map)) => map.is_empty(), + _ => false, + } +} + #[cfg(feature = "parquet")] async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result> { let gcs_resource = gcs_resource_ref.clone(); @@ -509,7 +526,12 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result
@@ -1129,6 +1436,27 @@ + + { + const it = createConfirm + createConfirm = undefined + if (it) createOnRemote(it) + }} + onCanceled={() => (createConfirm = undefined)} + > +

+ This copies the current value of {createConfirm?.path} + (including any secret value) from + {createConfirm?.onCurrent ? currentWorkspaceId : parentWorkspaceId} + into {createConfirm?.onCurrent ? parentWorkspaceId : currentWorkspaceId}. It stays + workspace-specific afterward, so later promotes won't overwrite it. If it already exists + there, it's left untouched and just marked workspace-specific. +

+ {:else}
No comparison data available
diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte new file mode 100644 index 0000000000..bdba23c57f --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -0,0 +1,191 @@ + + +{#if isDev && parentId} +
+

+ This is a dev workspace paired with root workspace {parentId}. Promote changes + from the home page banner or the Compare & Deploy page. +

+
+ +
+
+{:else if pairedDev} +
+

+ This workspace's dev workspace is {pairedDev.name} ({pairedDev.id}). Edits to this + workspace are redirected there. +

+
+ {#if pairedDev.isMember} + + {/if} + +
+
+{:else if parentId} +

+ Dev workspace pairing is only available for root workspaces. This workspace is a fork of + {parentId}. +

+{:else} +
+

+ Pair this workspace with a dev workspace: the same code with a different environment (resource + and variable values). Edits are made in the dev workspace and promoted here. +

+
+ Attach an existing workspace as dev + Date: Thu, 2 Jul 2026 11:14:19 +0200 Subject: [PATCH 38/56] feat(frontend): pipelines index page and sql editor hint (#9881) * feat(frontend): surface pipelines in sidebar nav, index page and sql editor hint Co-Authored-By: Claude Fable 5 * feat(frontend): remove pipelines sidebar nav item Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../src/lib/components/ScriptBuilder.svelte | 55 +++++++++ .../AssetGraph/PipelineFolderList.svelte | 111 ++++++++++++++++++ .../AssetGraph/PipelinePickerModal.svelte | 93 +-------------- .../(root)/(logged)/pipeline/+page.svelte | 63 +++++----- 4 files changed, 204 insertions(+), 118 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 36ac67559b..fbfa4f1c1b 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -56,6 +56,7 @@ Code, DiffIcon, EllipsisVertical, + Network, Plus, Rocket, Settings, @@ -63,6 +64,9 @@ Tag, X } from 'lucide-svelte' + import { base } from '$lib/base' + import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' + import { parsePipelineAnnotations } from './assets/AssetGraph/parsePipelineAnnotations' import DropdownV2 from './DropdownV2.svelte' import { type Item } from '$lib/utils' import { sendUserToast } from '$lib/toast' @@ -314,6 +318,21 @@ const enterpriseLangs = ['bigquery', 'snowflake', 'mssql', 'oracledb'] + // Languages the pipeline editor treats as warehouse/dataset transforms — + // the ones where a `-- pipeline` annotation is a natural next step. + const pipelineHintLangs = ['duckdb', 'postgresql', 'bigquery', 'snowflake', 'mysql', 'mssql'] + const pipelineHintDismissed = useLocalStorageValue( + 'pipelineScriptHintDismissed', + false, + 'boolean' + ) + let showPipelineHint = $derived( + !pipelineHintDismissed.val && + (script.kind === 'script' || script.kind === undefined) && + pipelineHintLangs.includes(script.language ?? '') && + !parsePipelineAnnotations(script.content ?? '').inPipeline + ) + export function setCode(code: string): void { editor?.setCode(code) } @@ -1993,6 +2012,42 @@
+ {#if showPipelineHint} +
+ + + This script can become a data pipeline step: annotate it with + -- pipeline + and + + -- materialize + + to materialize its result, or build it in the + pipeline editor. + + Learn more + + +
+
+
+ {/if} + + import { userStore, workspaceStore } from '$lib/stores' + import { base } from '$lib/base' + import { goto } from '$app/navigation' + import Button from '$lib/components/common/button/Button.svelte' + import FolderPicker from '$lib/components/FolderPicker.svelte' + import { AssetService, type ListPipelineFoldersResponse } from '$lib/gen' + import { resource } from 'runed' + import { ArrowRight, Loader2 } from 'lucide-svelte' + + interface Props { + // When provided, the current folder is dropped from the existing list + // so users don't "switch" to the folder they're already on. + currentFolder?: string | undefined + // Carried over to the target URL so switching folders from the + // editor keeps the user in edit mode; the default view mode needs + // no param. + mode?: 'view' | 'edit' + // Called right before navigating — lets a wrapping modal close itself. + onOpen?: () => void + } + let { currentFolder = undefined, mode = 'view', onOpen = undefined }: Props = $props() + + let pipelines = resource( + () => $workspaceStore, + async (ws) => { + if (!ws) return [] as ListPipelineFoldersResponse + return await AssetService.listPipelineFolders({ workspace: ws }) + } + ) + + let pickedFolder = $state('') + + let visiblePipelines = $derived( + (pipelines.current ?? []).filter((p) => p.folder !== currentFolder) + ) + + async function openExistingPipeline(folder: string) { + onOpen?.() + const modeQuery = mode !== 'view' ? `?mode=${mode}` : '' + await goto(`${base}/pipeline/${encodeURIComponent(folder)}${modeQuery}`) + } + + async function openPicked() { + const name = pickedFolder.trim() + if (!name) return + await openExistingPipeline(name) + } + + +
+
+

+ Existing pipelines +

+ {#if pipelines.loading && !pipelines.current} +
+ + Loading… +
+ {:else if pipelines.error} +
Failed: {pipelines.error.message}
+ {:else if visiblePipelines.length === 0} +
+ {currentFolder + ? 'No other pipelines in this workspace.' + : 'No pipelines yet. A pipeline is any folder whose scripts carry pipeline annotations.'} +
+ {:else} +
+ {#each visiblePipelines as p (p.folder)} + + {/each} +
+ {/if} +
+ + {#if !$userStore?.operator} + +
+

+ Pick or create a folder +

+
+
+ +
+ +
+
+ {/if} +
diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte index ded1013e25..b2183eec77 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte @@ -1,13 +1,6 @@ -
- {#if visiblePipelines.length > 0 || pipelines.loading} -
-

- Existing pipelines -

- {#if pipelines.loading && !pipelines.current} -
- - Loading… -
- {:else if pipelines.error} -
Failed: {pipelines.error.message}
- {:else} -
- {#each visiblePipelines as p} - - {/each} -
- {/if} -
- {/if} - - {#if !$userStore?.operator} - -
-

- Pick or create a folder -

-
-
- -
- -
-
- {/if} -
+ (open = false)} /> diff --git a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte index 98f598165e..ab36b4c1f7 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte @@ -1,24 +1,20 @@ - Pipeline editor — Windmill + Pipelines — Windmill
@@ -45,28 +40,42 @@ >
-

- {$userStore?.operator ? 'Pipelines' : 'Pipeline editor'} -

- · no folder selected +

Pipelines

+ + Alpha +
-
-
- - Pick a folder to open its pipeline. - +
+
+
+

Data pipelines

+

+ Chain ingestion, transformation and materialization steps into an asset-aware graph. + {#if !$userStore?.operator} + Each pipeline lives in a folder: open one below, or pick a folder to start a new + pipeline. + {:else} + Open a pipeline below to view its graph and runs. + {/if} +

+ + + Pipelines documentation + +
+ +
- From b92a86b8b3a60b877540c3a7f0ffefe36ccbb053 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 11:48:11 +0200 Subject: [PATCH 39/56] fix: polish pipeline graph view (layout, viewport, minimap, lineage, timestamps) (#9883) * fix(frontend): keep pipeline graph layered when lineage has cycles Co-Authored-By: Claude Fable 5 * fix(frontend): fit pipeline graph to visible canvas on initial load Co-Authored-By: Claude Fable 5 * fix(frontend): style pipeline minimap so it reads as a minimap Co-Authored-By: Claude Fable 5 * fix(parser): don't infer s3 reads from bare string-literal mentions in sql Co-Authored-By: Claude Fable 5 * fix(duckdb): render temporal values as ISO strings in job results Co-Authored-By: Claude Fable 5 * fix(frontend): key pipeline viewport fit on the loaded graph's folder Co-Authored-By: Claude Fable 5 * fix(parser): treat list/named read-fn arguments as definitive s3 reads Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../src/asset_parser.rs | 220 ++++++++++++++++-- .../windmill-duckdb-ffi-internal/src/lib.rs | 114 ++++++++- .../assets/AssetGraph/AssetGraphCanvas.svelte | 50 +++- .../assets/AssetGraph/InitialFitView.svelte | 30 +++ .../AssetGraph/PipelineGraphEditor.svelte | 9 + .../AssetGraph/assetGraphLayout.test.ts | 40 +++- .../assets/AssetGraph/assetGraphLayout.ts | 37 ++- .../sessions/PipelineEditorView.svelte | 10 + .../(logged)/pipeline/[folder]/+page.svelte | 11 + 9 files changed, 482 insertions(+), 39 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index f7c578a274..c4f93efdb7 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -83,18 +83,56 @@ pub fn parse_assets(input: &str) -> anyhow::Result { // Body-inferred column lineage, with `// column` annotations taking // precedence per output column (explicit declaration overrides inference). pipeline.column_lineage = merge_column_lineage(inferred, pipeline.column_lineage); - Ok(ParseAssetsOutput::new( - merge_assets(collector.assets), - Vec::new(), - pipeline, - )) + // A bare string literal in query position is only weak read evidence: a + // summary `SELECT 's3:///out.csv' AS target` after `COPY … TO + // 's3:///out.csv'` must not turn the write into rw (which draws a + // spurious read edge and an asset⇄script cycle in the pipeline graph). + // Surface weak reads only for assets with no other recorded usage, so a + // path that is *merely* mentioned still shows up linked to the script. + let mut assets = merge_assets(collector.assets); + for weak in merge_assets(collector.weak_string_reads) { + if !assets + .iter() + .any(|a| a.kind == weak.kind && a.path == weak.path) + { + assets.push(weak); + } + } + assets.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(ParseAssetsOutput::new(assets, Vec::new(), pipeline)) +} + +/// Provenance of the innermost access context. The access type alone can't +/// tell a definitive read apart from a mere mention: a bare string literal in +/// generic query position (`QueryRead`) is only *weak* read evidence — e.g. a +/// summary `SELECT 's3:///out.csv' AS target` echoing a path — while the same +/// literal as a read-function argument or a `COPY` target is definitive. +#[derive(Clone, Copy, PartialEq, Eq)] +enum AccessCtx { + QueryRead, + ReadFn, + CopyWrite, +} + +impl AccessCtx { + fn access_type(self) -> AssetUsageAccessType { + match self { + AccessCtx::QueryRead | AccessCtx::ReadFn => R, + AccessCtx::CopyWrite => W, + } + } } /// Visitor that collects S3 asset literals from SQL statements struct AssetCollector { assets: Vec, - // e.g set to Read when we are inside a SELECT ... FROM ... statement - current_access_type_stack: Vec, + // Bare string literals seen in generic query position — weak read + // evidence, surfaced by `parse_assets` only when the script has no other + // recorded usage of the same asset (a real write must not gain a spurious + // read edge from a mention). + weak_string_reads: Vec, + // e.g set to QueryRead when we are inside a SELECT ... FROM ... statement + current_access_type_stack: Vec, // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } var_identifiers: BTreeMap, // e.g USE dl; @@ -119,6 +157,7 @@ impl AssetCollector { fn new() -> Self { Self { assets: Vec::new(), + weak_string_reads: Vec::new(), current_access_type_stack: Vec::with_capacity(8), var_identifiers: BTreeMap::new(), currently_used_asset: None, @@ -162,7 +201,11 @@ impl AssetCollector { name: &ObjectName, access_type: Option, ) -> Option { - let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied()); + let access_type = access_type.or_else(|| { + self.current_access_type_stack + .last() + .map(|c| c.access_type()) + }); if let Some((kind, path)) = &self.currently_used_asset { // We don't want to infer that any simple identifier refers to an asset if // we are not in a known R/W context @@ -301,12 +344,18 @@ impl AssetCollector { // Check if the string matches our asset syntax patterns if let Some((kind, path)) = parse_asset_syntax(s, false) { if kind == AssetKind::S3Object { - self.assets.push(ParseAssetsResult { + let ctx = self.current_access_type_stack.last().copied(); + let result = ParseAssetsResult { kind, path: path.to_string(), - access_type: self.current_access_type_stack.last().copied(), + access_type: ctx.map(AccessCtx::access_type), columns: None, - }); + }; + if ctx == Some(AccessCtx::QueryRead) { + self.weak_string_reads.push(result); + } else { + self.assets.push(result); + } } } } @@ -314,7 +363,7 @@ impl AssetCollector { fn handle_obj_name_pre(&mut self, name: &ObjectName) { if let Some(fname) = get_trivial_obj_name(name) { if is_read_fn(fname) { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::ReadFn); } } if let Some(str_lit) = get_str_lit_from_obj_name(name) { @@ -691,9 +740,20 @@ impl Visitor for AssetCollector { match table_factor { TableFactor::Table { name, args, .. } => { if args.is_none() { - // Avoid Table Functions - self.handle_obj_name_pre(name); + // FROM 's3:///…' is a definitive read — record it directly + // so it isn't demoted to a weak in-query mention. + if let Some(asset) = self.get_s3_asset_from_str_literal_table(table_factor) { + self.assets.push(asset); + } } + // For a read-function table factor this pushes ReadFn, making + // every literal inside its arguments a definitive read — the + // direct form (read_csv('s3:///…')) but also list and named + // arguments (read_parquet(['s3:///…'])). Must run for BOTH the + // plain-table and table-function branches: post_visit_table_factor + // pops via handle_obj_name_post unconditionally, so skipping the + // push here would unbalance the stack. + self.handle_obj_name_pre(name); } _ => {} } @@ -719,6 +779,13 @@ impl Visitor for AssetCollector { Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => { self.handle_string_literal(s); } + // Read-function call in expression position: its argument literals + // are definitive reads. Balances the pop in `post_visit_expr`. + Expr::Function(func) => { + if get_trivial_obj_name(&func.name).is_some_and(is_read_fn) { + self.current_access_type_stack.push(AccessCtx::ReadFn); + } + } _ => {} } std::ops::ControlFlow::Continue(()) @@ -946,7 +1013,7 @@ impl Visitor for AssetCollector { } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { - self.current_access_type_stack.push(W); + self.current_access_type_stack.push(AccessCtx::CopyWrite); self.handle_string_literal(filename); self.current_access_type_stack.pop(); } @@ -1013,7 +1080,7 @@ impl Visitor for AssetCollector { &mut self, query: &sqlparser::ast::Query, ) -> std::ops::ControlFlow { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::QueryRead); self.cte_name_stack.push(collect_cte_names(query)); std::ops::ControlFlow::Continue(()) } @@ -1098,6 +1165,127 @@ mod tests { ); } + #[test] + fn test_copy_target_echoed_in_select_stays_write_only() { + // The trailing summary SELECT merely mentions the COPY target — it + // must not add a read (rw would draw an asset⇄script cycle). + let input = r#" + COPY (SELECT 1 AS x) TO 's3:///out.csv'; + SELECT 's3:///out.csv' AS target, 42 AS rows_written; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/out.csv".to_string(), + access_type: Some(W), + columns: None + }]) + ); + } + + #[test] + fn test_bare_string_mention_without_other_usage_is_a_read() { + let input = r#" + SELECT 's3:///referenced.csv' AS path; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/referenced.csv".to_string(), + access_type: Some(R), + columns: None + }]) + ); + } + + #[test] + fn test_self_refresh_read_fn_plus_copy_stays_rw() { + // A definitive read (read_csv) of the same file the script rewrites + // is a real rw — only *bare-literal* mentions are demoted. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_csv('s3:///data.csv'); + COPY (SELECT * FROM tmp) TO 's3:///data.csv'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.csv".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_plus_copy_stays_rw() { + // read_parquet's list form is as definitive as the direct literal — + // it must not be demoted to a weak mention when the file is rewritten. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_parquet(['s3:///data.parquet']); + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_multiple_files_are_reads() { + let input = r#" + SELECT * FROM read_parquet(['s3:///a.parquet', 's3:///b.parquet']); + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/a.parquet".to_string(), + access_type: Some(R), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/b.parquet".to_string(), + access_type: Some(R), + columns: None + } + ]) + ); + } + + #[test] + fn test_from_string_literal_of_written_file_stays_rw() { + // FROM-position string literal is likewise a definitive read. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM 's3:///data.parquet'; + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + #[test] fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() { let input = r#" diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index f3ee59cab7..054f892220 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -635,7 +635,9 @@ fn duckdb_value_to_json_value( .ok_or_else(|| "Could not convert to f64".to_string())?, ), duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Timestamp(unit, ts) => { + serde_json::Value::String(duckdb_timestamp_to_iso(unit, ts)) + } duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { serde_json::from_str(&s) .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? @@ -646,8 +648,15 @@ fn duckdb_value_to_json_value( .map(|byte| serde_json::Value::Number(byte.into())) .collect(), ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Date32(d) => { + match chrono::DateTime::from_timestamp(i64::from(d) * 86_400, 0) { + Some(dt) => serde_json::Value::String(dt.date_naive().to_string()), + None => serde_json::Value::Number(d.into()), + } + } + duckdb::types::Value::Time64(unit, t) => { + serde_json::Value::String(duckdb_time_to_iso(unit, t).unwrap_or_else(|| t.to_string())) + } duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ "months": months, "days": days, @@ -688,6 +697,105 @@ fn duckdb_value_to_json_value( Ok(json_value) } +// DuckDB surfaces TIMESTAMP[_S/_MS/_NS] values as a raw count since the epoch; +// stringifying that count leaks values like "1782974022218435" into job +// results. Render ISO-8601 instead (the Postgres executor's +// "2024-01-15T10:30:00" shape); a count outside chrono's representable range +// falls back to the raw number. +fn duckdb_timestamp_to_iso(unit: duckdb::types::TimeUnit, ts: i64) -> String { + let dt = match unit { + duckdb::types::TimeUnit::Second => chrono::DateTime::from_timestamp(ts, 0), + duckdb::types::TimeUnit::Millisecond => chrono::DateTime::from_timestamp_millis(ts), + duckdb::types::TimeUnit::Microsecond => chrono::DateTime::from_timestamp_micros(ts), + duckdb::types::TimeUnit::Nanosecond => Some(chrono::DateTime::from_timestamp_nanos(ts)), + }; + dt.map(|dt| dt.naive_utc().format("%Y-%m-%dT%H:%M:%S%.f").to_string()) + .unwrap_or_else(|| ts.to_string()) +} + +// Same story for TIME: a raw count since midnight. None when out of range +// (caller falls back to the raw number). +fn duckdb_time_to_iso(unit: duckdb::types::TimeUnit, t: i64) -> Option { + let (secs, nanos) = match unit { + duckdb::types::TimeUnit::Second => (t, 0), + duckdb::types::TimeUnit::Millisecond => (t / 1_000, (t % 1_000) * 1_000_000), + duckdb::types::TimeUnit::Microsecond => (t / 1_000_000, (t % 1_000_000) * 1_000), + duckdb::types::TimeUnit::Nanosecond => (t / 1_000_000_000, t % 1_000_000_000), + }; + chrono::NaiveTime::from_num_seconds_from_midnight_opt( + u32::try_from(secs).ok()?, + u32::try_from(nanos).ok()?, + ) + .map(|t| t.format("%H:%M:%S%.f").to_string()) +} + +#[cfg(test)] +mod temporal_json_tests { + use super::*; + use duckdb::types::TimeUnit; + + #[test] + fn timestamp_micros_renders_iso() { + // 2026-07-01 23:13:42.218435 UTC + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Microsecond, 1_782_947_622_218_435), + "2026-07-01T23:13:42.218435" + ); + } + + #[test] + fn timestamp_seconds_renders_iso_without_subseconds() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, 1_735_689_600), + "2025-01-01T00:00:00" + ); + } + + #[test] + fn out_of_range_timestamp_falls_back_to_raw() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, i64::MAX), + i64::MAX.to_string() + ); + } + + #[test] + fn time_micros_renders_iso() { + assert_eq!( + duckdb_time_to_iso(TimeUnit::Microsecond, 37_800_500_000).as_deref(), + Some("10:30:00.500") + ); + } + + #[test] + fn temporal_values_render_iso_through_real_query() { + let conn = duckdb::Connection::open_in_memory().unwrap(); + let mut stmt = conn + .prepare( + "SELECT TIMESTAMP '2026-07-01 23:13:42.218435' AS ts, + TIMESTAMPTZ '2026-07-01 23:13:42+00' AS tstz, + TIMESTAMP_NS '2026-07-01 23:13:42.218435678' AS ts_ns, + DATE '2026-07-01' AS d, + TIME '10:30:00' AS t", + ) + .unwrap(); + let mut rows = stmt.query([]).unwrap(); + let row = rows.next().unwrap().unwrap(); + let json_of = |i: usize| { + let v: duckdb::types::Value = row.get(i).unwrap(); + duckdb_value_to_json_value(v, &None).unwrap() + }; + assert_eq!(json_of(0), serde_json::json!("2026-07-01T23:13:42.218435")); + assert_eq!(json_of(1), serde_json::json!("2026-07-01T23:13:42")); + assert_eq!( + json_of(2), + serde_json::json!("2026-07-01T23:13:42.218435678") + ); + assert_eq!(json_of(3), serde_json::json!("2026-07-01")); + assert_eq!(json_of(4), serde_json::json!("10:30:00")); + } +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 153d7c2965..6851fc843a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -16,6 +16,7 @@ import DataTestNode from './DataTestNode.svelte' import AssetGraphEdge from './AssetGraphEdge.svelte' import PanToNode from './PanToNode.svelte' + import InitialFitView from './InitialFitView.svelte' import { layoutAssetGraph } from './assetGraphLayout' import { buildDownstreamMap } from './graphTraversal' import { buildLineageDownstreamMap } from './boundedCascade' @@ -162,6 +163,10 @@ /** Hide the minimap when the canvas is too narrow for it to be worth the * space (e.g. stacked layout in a side panel). Defaults to shown. */ showMinimap?: boolean + /** Identity of the displayed graph (e.g. the pipeline folder). The + * initial viewport fit re-arms when it changes, so switching folders + * in-place gets a fresh fit. */ + viewportFitKey?: string } let { graph, @@ -188,7 +193,8 @@ onStartBoundedRun, boundPick, onPickEnd, - showMinimap = true + showMinimap = true, + viewportFitKey = '' }: Props = $props() // `${kind}:${path}` ids for the hovered / pinned runs (both script and flow @@ -677,11 +683,27 @@ // edges + paths → same layout. Renames *do* change the layout for // the renamed entry, since its id moves in the sort, but that's // expected: a rename is a path change, which is part of the input. + // A script with rw access to an asset yields both a write (script → asset) + // and a read/trigger (asset → script) edge — a 2-cycle. The layout resolves + // it producer-above-asset by omitting the backward direction from its + // input; the rendered edges are untouched (both arrows still drawn). + let writeEdgePairs = $derived( + new Set( + model.edges.filter((e) => e.kind === 'lineage-write').map((e) => `${e.source}\n${e.target}`) + ) + ) let layoutInput = $derived({ nodes: model.nodes .map((n) => ({ id: n.id, data: n.data })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), edges: model.edges + .filter( + (e) => + !( + (e.kind === 'lineage-read' || e.kind === 'trigger-asset') && + writeEdgePairs.has(`${e.target}\n${e.source}`) + ) + ) .map((e) => ({ source: e.source, target: e.target })) .sort((a, b) => a.source === b.source @@ -1023,21 +1045,35 @@ --background-color={false} >
+ {#if showMinimap} + n.type === 'asset' - ? 'rgb(96 165 250 / 0.5)' + ? 'rgb(59 130 246 / 0.3)' : n.type === 'trigger' - ? 'rgb(251 191 36 / 0.5)' - : 'rgb(52 211 153 / 0.5)'} - nodeStrokeColor="transparent" - maskColor="rgb(0 0 0 / 0.2)" + ? 'rgb(245 158 11 / 0.3)' + : 'rgb(148 163 184 / 0.15)'} + nodeStrokeColor={(n) => + n.type === 'asset' + ? 'rgb(59 130 246 / 0.8)' + : n.type === 'trigger' + ? 'rgb(245 158 11 / 0.8)' + : 'rgb(100 116 139 / 0.7)'} + maskColor="rgb(100 116 139 / 0.12)" + maskStrokeColor="rgb(59 130 246 / 0.5)" + maskStrokeWidth={4} /> {/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte b/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte new file mode 100644 index 0000000000..eea1fb306d --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte @@ -0,0 +1,30 @@ + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte index 33d44eb7a4..8dca18bd8e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte @@ -31,6 +31,7 @@ mode, workspace, folder, + viewportFitKey = undefined, stackBelow = 680, persistDrafts = false, pathPrefix, @@ -95,6 +96,13 @@ workspace: string | undefined /** Folder the pipeline is scoped to — drives the autosave draft path. */ folder: string + /** Folder whose graph is actually *loaded* (not the route param). On an + * in-place folder switch the stale graph stays rendered while the new + * fetch is in flight, so keying the canvas's one-shot initial fit on + * `folder` would consume the new folder's fit on the old graph. Pages + * that stale-while-revalidate should pass the folder captured when the + * fetch resolved; defaults to `folder`. */ + viewportFitKey?: string /** Below this container width (px) the graph/details split stacks * vertically instead of side-by-side — for narrow side panels / AI * session previews. Defaults to 680. */ @@ -433,6 +441,7 @@ {onPickEnd} {panToNodeId} showMinimap={!stacked} + viewportFitKey={viewportFitKey ?? folder} /> {#if boundBar}{@render boundBar()}{/if} {#if mode === 'edit'} diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts index a5a92f61c2..24e7d4c2da 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts @@ -38,7 +38,14 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { // stays strictly left of every node of the right branch. const pos = layoutAssetGraph({ nodes: [n('root'), n('a'), n('b'), n('a1'), n('a2'), n('a3'), n('b1')], - edges: [e('root', 'a'), e('root', 'b'), e('a', 'a1'), e('a', 'a2'), e('a', 'a3'), e('b', 'b1')] + edges: [ + e('root', 'a'), + e('root', 'b'), + e('a', 'a1'), + e('a', 'a2'), + e('a', 'a3'), + e('b', 'b1') + ] }) const leftMax = Math.max(...['a', 'a1', 'a2', 'a3'].map((id) => pos.get(id)!.x)) const rightMin = Math.min(...['b', 'b1'].map((id) => pos.get(id)!.x)) @@ -83,14 +90,37 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => { } }) - it('falls back to a grid on cyclic input', () => { + it('lays a 2-cycle out as a chain (feedback edge dropped, no grid)', () => { const pos = layoutAssetGraph({ nodes: [n('a'), n('b')], edges: [e('a', 'b'), e('b', 'a')] }) - expect(pos.size).toBe(2) - expect(pos.get('a')).toBeDefined() - expect(pos.get('b')).toBeDefined() + // First-in-input wins the top slot; the b→a feedback edge is ignored. + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('a')!.x).toBe(pos.get('b')!.x) + }) + + it('keeps the acyclic part of a graph layered when one cycle exists', () => { + // root → a ⇄ b → leaf: the a⇄b cycle must not degrade root/leaf layering. + const pos = layoutAssetGraph({ + nodes: [n('root'), n('a'), n('b'), n('leaf')], + edges: [e('root', 'a'), e('a', 'b'), e('b', 'a'), e('b', 'leaf')] + }) + expect(pos.get('root')!.y).toBeLessThan(pos.get('a')!.y) + expect(pos.get('a')!.y).toBeLessThan(pos.get('b')!.y) + expect(pos.get('b')!.y).toBeLessThan(pos.get('leaf')!.y) + // A linear chain stays in one column. + expect(new Set(['root', 'a', 'b', 'leaf'].map((id) => pos.get(id)!.x)).size).toBe(1) + }) + + it('handles a longer cycle without dropping nodes', () => { + const pos = layoutAssetGraph({ + nodes: [n('a'), n('b'), n('c')], + edges: [e('a', 'b'), e('b', 'c'), e('c', 'a')] + }) + expect(pos.size).toBe(3) + const ys = ['a', 'b', 'c'].map((id) => pos.get(id)!.y) + expect(new Set(ys).size).toBe(3) }) it('packs disjoint components side by side without overlap', () => { diff --git a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts index cf55db6e0a..c9d7d60355 100644 --- a/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts +++ b/frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts @@ -53,8 +53,8 @@ interface Band { // // y comes from longest-path layering (same top-down orientation as before: // producers above, assets in the middle, consumers below). Returns positions -// (band centers) normalized so the component's min x,y = 0. Throws on cyclic -// input (caller falls back to a grid for the whole graph). +// (band centers) normalized so the component's min x,y = 0. Cyclic input is +// handled by dropping feedback edges (see the Kahn step below). function layoutComponent( nodes: GraphInput['nodes'], edges: GraphInput['edges'] @@ -76,21 +76,42 @@ function layoutComponent( if (!parents.get(e.target)!.includes(e.source)) parents.get(e.target)!.push(e.source) } - // Kahn topological order — also the cycle guard. + // Kahn topological order. Cycles don't abort the layout: when the queue + // drains with nodes left, the unplaced node with the fewest outstanding + // parents (first in input order on ties) is forced into the order and its + // not-yet-placed parent edges are dropped as feedback edges — layering and + // tree-building then operate on the resulting DAG while the rendered graph + // keeps every edge. (The caller already resolves write⇄read 2-cycles by + // omitting the read direction; this handles any longer cycle.) const indeg = new Map() for (const n of nodes) indeg.set(n.id, parents.get(n.id)!.length) const queue = nodes.filter((n) => indeg.get(n.id) === 0).map((n) => n.id) + const placed = new Set() const topo: string[] = [] - while (queue.length) { + while (topo.length < nodes.length) { + if (queue.length === 0) { + let pick: string | undefined + for (const n of nodes) { + if (placed.has(n.id)) continue + if (pick === undefined || indeg.get(n.id)! < indeg.get(pick)!) pick = n.id + } + parents.set( + pick!, + parents.get(pick!)!.filter((p) => placed.has(p)) + ) + queue.push(pick!) + } const cur = queue.shift()! + if (placed.has(cur)) continue + placed.add(cur) topo.push(cur) for (const c of children.get(cur)!) { + if (placed.has(c)) continue const d = indeg.get(c)! - 1 indeg.set(c, d) if (d === 0) queue.push(c) } } - if (topo.length !== nodes.length) throw new Error('cyclic asset graph') // Longest-path layering: a node sits one layer below its lowest parent. const layer = new Map() @@ -141,8 +162,7 @@ function layoutComponent( out.set(id, { x: left + w / 2, y: layer.get(id)! * LAYER_H }) const kids = treeChildren.get(id)! if (kids.length === 0) return - const kidsW = - kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) + const kidsW = kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1) let cursor = left + (w - kidsW) / 2 for (const k of kids) { placeTree(k, cursor) @@ -221,7 +241,8 @@ function layoutComponent( // disjoint components, so it's excluded from component detection and instead // re-placed centered one layer above the whole packed graph. // -// Falls back to a stable grid if the component layout throws (cyclic inputs). +// Falls back to a stable grid if the component layout throws (defensive — +// cycles are already absorbed by feedback-edge dropping in layoutComponent). export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map { const byId = new Map() if (graph.nodes.length === 0) return byId diff --git a/frontend/src/lib/components/sessions/PipelineEditorView.svelte b/frontend/src/lib/components/sessions/PipelineEditorView.svelte index e0578ae139..0e3be64e36 100644 --- a/frontend/src/lib/components/sessions/PipelineEditorView.svelte +++ b/frontend/src/lib/components/sessions/PipelineEditorView.svelte @@ -75,6 +75,15 @@ workspace && folder ? await AssetService.getAssetsGraph({ workspace, folder }) : EMPTY_GRAPH ) + // Folder whose graph is actually rendered — `graphRes.current` is stale- + // while-revalidate on a folder retarget, so the canvas's one-shot initial + // fit is keyed on the folder captured when a graph lands, not on `path` + // (same rationale as the pipeline route page). + let viewportFitFolder = $state('') + $effect(() => { + if (graphRes.current) untrack(() => (viewportFitFolder = path)) + }) + // Deployed graph + the in-flight draft overlay (AI-built nodes render as plain // dashed unsaved drafts, same as manual drafts). The session // skips the route page's folder-wide asset prefetch (empty inferred maps); the @@ -304,6 +313,7 @@ { + if (graphRes.current) untrack(() => (viewportFitFolder = folder)) + }) + // Body / inferred-assets prefetch sweep. Watches `g.runnables`; for any // non-draft path we haven't fetched yet, fetches `getScriptByPath` and // `inferAssets`, and stores both in their respective only-add caches. @@ -2121,6 +2131,7 @@ Date: Thu, 2 Jul 2026 11:58:31 +0200 Subject: [PATCH 40/56] reserve more fixed height for unsaved-changes banner to avoid content shift (#9873) * fix(frontend): reserve fixed height for unsaved-changes banner to avoid content shift Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): remove border around reserved banner slot and shrink it Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): tighten top padding under the unsaved-changes banner Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): give unsaved-changes banner buttons minimal vertical breathing room Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(frontend): drop redundant Metadata section title in trigger and script editors Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): tuck schedule editor labels under summary to match convention Co-Authored-By: Claude Opus 4.8 (1M context) * revert(frontend): keep Metadata section title in ScriptBuilder for a separate PR Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): drop leftover header-content margin on headless Section Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): drop top padding above resource editor first field Co-Authored-By: Claude Opus 4.8 (1M context) * style(frontend): match variable editor bottom padding to resource Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): add Path label in new resource form to match edit Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reserve half the banner height to halve the idle gap Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reserve a third of the banner height when idle Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): don't reserve banner slot or tighten top for new entities Gate the reserved-height slot and the tight content top padding on the banner's baseline (bannerReserved) instead of merely on the banner snippet being present, so new-entity drawers keep normal top spacing and add no empty slot. Co-Authored-By: Claude Opus 4.8 (1M context) * docs(frontend): trim banner comments to the 4-line invariant limit Co-Authored-By: Claude Opus 4.8 (1M context) * docs(frontend): describe partial-reserve banner behavior accurately Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../src/lib/components/AppConnectInner.svelte | 32 ++++--- .../lib/components/LocalDraftBanner.svelte | 91 +++++++++++-------- .../src/lib/components/ResourceEditor.svelte | 2 +- .../components/ResourceEditorDrawer.svelte | 2 + frontend/src/lib/components/Section.svelte | 2 +- .../src/lib/components/VariableEditor.svelte | 4 +- .../common/drawer/DrawerContent.svelte | 44 ++++++--- .../azure/AzureTriggerEditorInner.svelte | 2 + .../email/EmailTriggerEditorInner.svelte | 4 +- .../triggers/gcp/GcpTriggerEditorInner.svelte | 2 + .../triggers/http/RouteEditorInner.svelte | 4 +- .../kafka/KafkaTriggerEditorInner.svelte | 2 + .../mqtt/MqttTriggerEditorInner.svelte | 2 + .../native/NativeTriggerEditor.svelte | 2 +- .../nats/NatsTriggerEditorInner.svelte | 2 + .../PostgresTriggerEditorInner.svelte | 2 + .../schedules/ScheduleEditorInner.svelte | 6 +- .../triggers/sqs/SqsTriggerEditorInner.svelte | 2 + .../triggers/useTriggerDraftSync.svelte.ts | 12 +++ .../WebsocketTriggerEditorInner.svelte | 2 + 20 files changed, 148 insertions(+), 73 deletions(-) diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 9edd80dd54..ef632f5738 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -991,13 +991,15 @@
{:else if step == 2 && manual}
- + {#if deployTo}
`). + path: String, + // Inclusive calendar-day range (YYYY-MM-DD), local to the producer's + // partition tz. + from: chrono::NaiveDate, + to: chrono::NaiveDate, +} + +#[derive(Serialize)] +struct PartitionInRange { + partition: String, + // `missing` | `running` | `materialized` | `failed` — `missing` means no + // materialization was ever recorded for the slice. + status: &'static str, +} + +#[derive(Serialize)] +struct PartitionsInRangeResponse { + // The pipeline script that materializes the asset (managed `// materialize` + // target, or a partitioned writer using the SDK helpers) — the runnable a + // backfill launches (with an explicit `partition` arg per slice). + producer_path: String, + partition_kind: String, + partitions: Vec, +} + +// Backfill range preview: every partition the producer's `// partitioned` spec +// expects in `[from, to]`, joined with what `materialized_partition` records — +// the missing/failed subset is the backfill worklist. The logic is in the +// `backfill` module pair: EE resolves and enumerates, the OSS stub errors +// (single-partition runs stay available everywhere; fanning out over a range +// is enterprise). +async fn list_partitions_in_range( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let res = backfill::partitions_in_range(&mut tx, &w_id, &q).await?; + tx.commit().await?; + Ok(Json(res)) +} + // Per-asset captured output schema versions for a ducklake asset (gap #2a) — // the schema-evolution history persisted after each managed `// materialize`. // Newest version first; materialization targets are ducklake-only in v1, so the diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 56893a8737..ebbc98e83e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0fcbb9132e..e7b5badb8c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -21498,6 +21498,42 @@ paths: items: $ref: "#/components/schemas/MaterializedPartition" + /w/{workspace}/assets/partitions_in_range: + get: + summary: List expected partitions of a ducklake asset in a date range with their materialization status (enterprise) + operationId: listAssetPartitionsInRange + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + - name: from + in: query + required: true + description: Inclusive range start (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + - name: to + in: query + required: true + description: Inclusive range end (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + responses: + "200": + description: expected partitions in range with per-slice status — the missing/failed subset is the backfill worklist + content: + application/json: + schema: + $ref: "#/components/schemas/PartitionsInRange" + /w/{workspace}/assets/asset_schemas: get: summary: List captured output-schema versions for a ducklake asset @@ -29692,6 +29728,27 @@ components: error: type: string nullable: true + PartitionsInRange: + type: object + required: [producer_path, partition_kind, partitions] + properties: + producer_path: + type: string + description: the pipeline script that materializes the asset (managed `// materialize` target, or a partitioned writer using the SDK helpers) — the runnable a backfill launches + partition_kind: + type: string + enum: [daily, hourly, weekly, monthly, dynamic] + partitions: + type: array + items: + type: object + required: [partition, status] + properties: + partition: + type: string + status: + type: string + enum: [missing, running, materialized, failed] AssetSchemaVersion: type: object required: [version, columns, captured_at] diff --git a/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte index 1826ff6b41..e4390fe503 100644 --- a/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/BackfillRangeDialog.svelte @@ -2,78 +2,217 @@ import { Button } from '$lib/components/common' import Modal from '$lib/components/common/modal/Modal.svelte' import DateInput from '$lib/components/DateInput.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { Loader2 } from 'lucide-svelte' + import { resource } from 'runed' + import { AssetService } from '$lib/gen' import { enterpriseLicense } from '$lib/stores' + import type { BackfillSliceState } from './backfillRun' - // Backfill re-runs the CE materialization once per partition in [from, to]. - // It is an enterprise feature (orchestration over a range); the dialog is - // only reachable when licensed, but we guard here too so the action can - // never fire in CE. + // Range backfill of a partitioned ducklake asset (enterprise). The dialog + // previews the expected partitions in [from, to] against what + // `materialized_partition` records (the missing/failed set is the + // worklist), then hands the worklist to the parent, which runs the + // producer once per slice (see `backfillRun.ts`). Progress state lives in + // the parent so a run survives closing the dialog. interface Props { // Controlled by the parent. `$bindable()` without a default per the // AGENTS.md ban on `$bindable(default)` for optional props. open?: boolean assetPath: string - // Invoked with the inclusive ISO date range; the parent performs the - // actual fan-out (EE backfill endpoint). - onBackfill: (from: string, to: string) => Promise + workspace: string + // Live per-slice states of the in-flight (or last finished) backfill. + slices?: BackfillSliceState[] + running?: boolean + // Cancellation was requested but the in-flight slice hasn't finished yet. + cancelRequested?: boolean + onStart: (producerPath: string, partitions: string[]) => void + onCancel: () => void + // Clear the finished run so the range picker shows again. + onReset: () => void } - let { open = $bindable(), assetPath, onBackfill }: Props = $props() + let { + open = $bindable(), + assetPath, + workspace, + slices, + running, + cancelRequested, + onStart, + onCancel, + onReset + }: Props = $props() let fromDate = $state(undefined) let toDate = $state(undefined) - let loading = $state(false) - let error = $state(undefined) + let onlyMissing = $state(true) - let canSubmit = $derived(!!$enterpriseLicense && !!fromDate && !!toDate && !loading) - - async function submit() { - if (!fromDate || !toDate) return - loading = true - error = undefined - try { - await onBackfill(fromDate, toDate) - open = false - } catch (e) { - error = e instanceof Error ? e.message : String(e) - } finally { - loading = false + let preview = resource( + [() => workspace, () => assetPath, () => fromDate, () => toDate, () => open ?? false], + async ([ws, path, from, to, isOpen]) => { + if (!isOpen || !ws || !path || !from || !to) return undefined + return await AssetService.listAssetPartitionsInRange({ workspace: ws, path, from, to }) } + ) + + let worklist = $derived.by(() => { + const parts = preview.current?.partitions ?? [] + const picked = onlyMissing + ? parts.filter((p) => p.status === 'missing' || p.status === 'failed') + : parts + return picked.map((p) => p.partition) + }) + let counts = $derived.by(() => { + const c = { missing: 0, materialized: 0, failed: 0, running: 0 } + for (const p of preview.current?.partitions ?? []) c[p.status]++ + return c + }) + let canStart = $derived( + !!$enterpriseLicense && !running && !preview.loading && worklist.length > 0 + ) + + // Backend errors come back as a plain-text body (windmill_common::error). + function errText(e: unknown): string { + const err = e as { body?: unknown; message?: string } + if (typeof err?.body === 'string' && err.body) return err.body + return err?.message ?? String(e) + } + + const previewChipClass: Record = { + missing: 'bg-amber-100 text-amber-800 dark:bg-amber-900/40 dark:text-amber-300', + failed: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300', + materialized: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300', + running: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300' + } + const sliceChipClass: Record = { + pending: 'bg-surface-secondary text-tertiary', + running: 'bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300', + success: 'bg-green-100 text-green-800 dark:bg-green-900/40 dark:text-green-300', + failure: 'bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300' } - open ?? false, (v) => (open = v)} title={`Backfill ${assetPath}`}> + open ?? false, (v) => (open = v)} + title={`Backfill ${assetPath}`} + cancelText="Close" +>
{#if !$enterpriseLicense}

Partition backfill is an enterprise feature. Materializing a single partition is available in the open-source edition; reprocessing a historical range requires an enterprise license.

+ {:else if slices?.length} + +

+ {#if running && cancelRequested} + Cancelling — waiting for the current partition to finish; the rest will not run. + {:else if running} + Materializing {slices.filter((s) => s.status === 'success' || s.status === 'failure') + .length}/{slices.length} partitions sequentially — each run gets its partition as an explicit + arg. + {:else} + Backfill finished: {slices.filter((s) => s.status === 'success').length} succeeded, + {slices.filter((s) => s.status === 'failure').length} failed, + {slices.filter((s) => s.status === 'pending').length} not run. + {/if} +

+
+ {#each slices as s (s.partition)} +
+ + {s.status} + + {s.partition} + {#if s.status === 'running'} + + {/if} + {#if s.error} + {s.error} + {/if} +
+ {/each} +
{:else}

- Re-runs the materialization for each partition in the range. Re-running a partition is - idempotent, so this is safe to repeat. + Re-runs the producing script once per partition in the range, each with an explicit + partition arg. Re-running a partition is idempotent, so this is + safe to repeat.

-
+
From - +
To - +
- {#if error} -

{error}

+ {#if preview.loading} +
+ Computing partitions in range… +
+ {:else if preview.error} +

{errText(preview.error)}

+ {:else if preview.current} +
+

+ {preview.current.partitions.length} + {preview.current.partition_kind} partitions in range — {counts.missing} missing, + {counts.failed} failed, {counts.materialized} materialized. Producer: + {preview.current.producer_path} +

+
+ {#each preview.current.partitions as p (p.partition)} + + {p.partition} + + {/each} +
+ +
{/if} {/if}
{#snippet actions()} - - + {#if running} + + {:else if slices?.length} + + + {:else} + + {/if} {/snippet} diff --git a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte index 4b7226e753..4be0bb8b6c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PartitionStatusGrid.svelte @@ -1,10 +1,12 @@
Materialized partitions -
- +
+ {#if backfillRunning} + + {:else if backfillSlices?.length} + + {/if} +
+ +
@@ -122,4 +196,14 @@
- + (backfillSlices = undefined)} +/> diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts new file mode 100644 index 0000000000..69fc6f9728 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest' +import { runBackfill, type BackfillSliceState } from './backfillRun' + +// Deterministic fake backend: launch resolves with `job:`, +// waitTerminal resolves per the `results` table (default success), recording +// launch order. +function fakeRunner(results: Record = {}) { + const launched: string[] = [] + return { + launched, + launch: async (partition: string) => { + launched.push(partition) + return `job:${partition}` + }, + waitTerminal: async (jobId: string) => results[jobId.slice(4)] ?? ('success' as const) + } +} + +describe('runBackfill', () => { + it('runs slices sequentially in order and reports ok', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(true) + expect(res.cancelled).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'success', 'success']) + expect(res.slices.map((s) => s.jobId)).toEqual([ + 'job:2026-06-26', + 'job:2026-06-27', + 'job:2026-06-29' + ]) + }) + + it('continues past a failed slice — each slice is independent', async () => { + const r = fakeRunner({ '2026-06-27': 'failure' }) + const res = await runBackfill({ + partitions: ['2026-06-26', '2026-06-27', '2026-06-29'], + launch: r.launch, + waitTerminal: r.waitTerminal + }) + expect(r.launched).toEqual(['2026-06-26', '2026-06-27', '2026-06-29']) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'failure', 'success']) + }) + + it('records a launch error as slice failure and keeps going', async () => { + const r = fakeRunner() + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + if (p === 'a') throw new Error('boom') + return r.launch(p) + }, + waitTerminal: r.waitTerminal + }) + expect(res.slices[0]).toMatchObject({ status: 'failure', error: 'boom' }) + expect(res.slices[1].status).toBe('success') + }) + + it('stops before the next launch when cancelled, leaving the rest pending', async () => { + const r = fakeRunner() + let done = 0 + const res = await runBackfill({ + partitions: ['a', 'b', 'c'], + launch: r.launch, + waitTerminal: async (id) => { + done++ + return r.waitTerminal(id) + }, + isCancelled: () => done >= 1 + }) + expect(r.launched).toEqual(['a']) + expect(res.cancelled).toBe(true) + expect(res.ok).toBe(false) + expect(res.slices.map((s) => s.status)).toEqual(['success', 'pending', 'pending']) + }) + + it('cancels a job whose launch raced the cancellation', async () => { + let cancelled = false + const cancelledJobs: string[] = [] + const res = await runBackfill({ + partitions: ['a', 'b'], + launch: async (p) => { + // The user clicks cancel while the launch request is in flight — + // there is no job id to cancel yet. + cancelled = true + return `job:${p}` + }, + waitTerminal: async () => 'failure', + isCancelled: () => cancelled, + cancelJob: async (id) => { + cancelledJobs.push(id) + } + }) + expect(cancelledJobs).toEqual(['job:a']) + expect(res.cancelled).toBe(true) + expect(res.slices.map((s) => s.status)).toEqual(['failure', 'pending']) + }) + + it('emits a snapshot per transition, never mutating earlier snapshots', async () => { + const r = fakeRunner() + const snapshots: BackfillSliceState[][] = [] + await runBackfill({ + partitions: ['a'], + launch: r.launch, + waitTerminal: r.waitTerminal, + onUpdate: (s) => snapshots.push(s) + }) + // initial pending, running, running+jobId, terminal + expect(snapshots.map((s) => s[0].status)).toEqual(['pending', 'running', 'running', 'success']) + expect(snapshots[1][0].jobId).toBeUndefined() + expect(snapshots[2][0].jobId).toBe('job:a') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts new file mode 100644 index 0000000000..51fbc9664b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/backfillRun.ts @@ -0,0 +1,83 @@ +// Client-side orchestration of a partition-range backfill (enterprise): one +// deployed run of the producing script per slice, launched with an explicit +// `partition` arg (the worker only resolves a partition when the arg is +// absent, so the caller-provided value wins). Slices run sequentially — +// concurrent materializations of the same ducklake table would contend on +// the catalog commit — and a failed slice does not stop the rest: each slice +// is independent, and the missing/failed set is simply the next worklist. +// +// Pure module (no Svelte runes) so the loop is unit-testable; reactive +// progress is delivered via `onUpdate` snapshots, mirroring +// `cascadeOrchestrator.ts`. + +export type BackfillSliceStatus = 'pending' | 'running' | 'success' | 'failure' + +export type BackfillSliceState = { + partition: string + status: BackfillSliceStatus + jobId?: string + error?: string +} + +export type BackfillRunOptions = { + /** Partition values to materialize, in run order. */ + partitions: string[] + /** Launch one run of the producer with the given partition arg; returns the job id. */ + launch: (partition: string) => Promise + /** Resolve once the job reaches a terminal state. */ + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + /** Snapshot of all slice states, emitted on every transition. */ + onUpdate?: (slices: BackfillSliceState[]) => void + /** Checked before each launch; a true stop leaves the remaining slices 'pending'. */ + isCancelled?: () => boolean + /** + * Cancel a job whose launch raced the cancellation — the cancel click had + * no job id to act on yet, so the loop cancels it as soon as the id + * arrives. Must not throw (the job may already be terminal). + */ + cancelJob?: (jobId: string) => Promise +} + +export type BackfillRunResult = { + /** True when every slice ran and succeeded. */ + ok: boolean + /** True when the loop stopped early on `isCancelled`. */ + cancelled: boolean + slices: BackfillSliceState[] +} + +export async function runBackfill(opts: BackfillRunOptions): Promise { + const { partitions, launch, waitTerminal, onUpdate, isCancelled, cancelJob } = opts + const slices: BackfillSliceState[] = partitions.map((partition) => ({ + partition, + status: 'pending' + })) + const emit = () => onUpdate?.(slices.map((s) => ({ ...s }))) + emit() + let cancelled = false + for (const slice of slices) { + if (isCancelled?.()) { + cancelled = true + break + } + slice.status = 'running' + emit() + try { + slice.jobId = await launch(slice.partition) + emit() + if (isCancelled?.() && cancelJob) { + await cancelJob(slice.jobId) + } + slice.status = await waitTerminal(slice.jobId) + } catch (e) { + slice.status = 'failure' + slice.error = e instanceof Error ? e.message : String(e) + } + emit() + } + return { + ok: !cancelled && slices.every((s) => s.status === 'success'), + cancelled, + slices + } +} From 5745dfc6ea69dcc15dbf70d548c706a8f2974d2a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 12:51:17 +0200 Subject: [PATCH 43/56] smooth local pipeline dogfooding (#9888) --- AGENTS.md | 1 + backend/CLAUDE.md | 5 +++ cli/src/commands/pipeline/docs.ts | 25 ++++++++++---- cli/src/commands/pipeline/localGraph.ts | 38 +++++++++++++++++++++- cli/test/pipeline_local_graph_unit.test.ts | 23 ++++++++++++- 5 files changed, 83 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index aead237e59..ac00741c49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Open-source platform for internal tools, workflows, API integrations, background ## Dev Environment - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) +- **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. - **Frontend**: `REMOTE=http://localhost:8000 npm run dev` from `frontend/` (port 3000+) - **DB**: `psql postgres://postgres:changeme@localhost:5432/windmill` - **Login**: `admin@windmill.dev` / `changeme` diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 9ff2c0be90..086e251962 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -6,3 +6,8 @@ - **DB schema**: `backend/summarized_schema.txt` - **API routes entry point**: `windmill-api/src/lib.rs` - **OpenAPI spec**: `windmill-api/openapi.yaml` +- **DuckDB local jobs**: build the dynamic FFI library before running DuckDB scripts locally: + ```bash + cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh + ``` + Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts index 57fb70618a..b360c8b802 100644 --- a/cli/src/commands/pipeline/docs.ts +++ b/cli/src/commands/pipeline/docs.ts @@ -157,8 +157,6 @@ export async function generatePipelineDocs( opts: GlobalOptions & { local?: boolean; defaultTs?: "bun" | "deno" }, folder: string, ) { - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); const f = folder.replace(/^f\//, "").replace(/\/$/, ""); // `docs` WRITES PIPELINE.md / AGENTS.md / CLAUDE.md under `f/`; a `..` // segment would escape the folder and clobber files elsewhere in the tree. @@ -170,9 +168,12 @@ export async function generatePipelineDocs( // defaultTs (from wmill.yaml) drives .ts → bun/deno inference for the local graph. const merged = await mergeConfigWithConfigFile(opts); + const workspace = opts.local ? undefined : await resolveWorkspace(opts); + if (!opts.local) await requireLogin(opts); + const graph = opts.local ? (await buildLocalPipelineGraph({ root, folder: f, defaultTs: merged.defaultTs })).graph - : await fetchDeployedGraph(workspace.workspaceId, f); + : await fetchDeployedGraph(workspace!.workspaceId, f); if (graph.runnables.length === 0) { // The deployed graph is empty — but a user/agent in a working tree may have @@ -198,10 +199,20 @@ export async function generatePipelineDocs( } let datatableSchemas: any[] = []; - try { - datatableSchemas = await wmill.listDataTableSchemas({ workspace: workspace.workspaceId }); - } catch (err: any) { - log.warn(colors.yellow(`Could not fetch datatable schemas: ${err.message}`)); + const hasExplicitWorkspace = + !!opts.workspace || + (!!opts.baseUrl && !!opts.token) || + (!!process.env["WM_WORKSPACE"] && + !!process.env["WM_TOKEN"] && + !!(process.env["BASE_INTERNAL_URL"] ?? process.env["BASE_URL"])); + if (!opts.local || hasExplicitWorkspace) { + try { + const schemaWorkspace = workspace ?? await resolveWorkspace(opts); + if (opts.local) await requireLogin(opts); + datatableSchemas = await wmill.listDataTableSchemas({ workspace: schemaWorkspace.workspaceId }); + } catch (err: any) { + log.warn(colors.yellow(`Could not fetch datatable schemas: ${err.message}`)); + } } const md = generatePipelineMarkdown(f, graph, datatableSchemas, !!opts.local); diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index ce556d1d8e..9d40a0e4b7 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -241,6 +241,30 @@ function fallbackParse(content: string, language: string): ParseAssetsRaw { return out; } +function recoverHeaderNativeTriggers(content: string, language: string): string[] { + const raw = commentPrefix(language); + const p = raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const out: string[] = []; + const seen = new Set(); + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (trimmed === "") continue; + if (!trimmed.startsWith(raw)) break; + const marker = line.match(new RegExp(`^\\s*${p}\\s*on\\s+(\\S+)\\s*$`)); + if (!marker) continue; + const kind = marker[1]; + if (!NATIVE_KINDS.has(kind) || seen.has(kind)) continue; + seen.add(kind); + out.push(kind); + } + return out; +} + +function normalizeRetry(retry: ParseAssetsRaw["retry"]): ParseAssetsRaw["retry"] { + if (!retry?.delay) return retry; + return { ...retry, delay: retry.delay.replace(/^delay=/, "") }; +} + // Comment prefix for `volume:` annotations. Deliberately NOT `commentPrefix` // above (which returns `--` for SQL): volume annotations are only recognized for // the languages the backend/frontend recognize them for — mirrors @@ -398,6 +422,8 @@ export async function buildLocalPipelineGraph(args: { for (const s of all) { const out = await inferScriptAssets(s.content, s.language); if (!out.in_pipeline) continue; // not a pipeline member + const retry = normalizeRetry(out.retry); + const nativeTriggers = recoverHeaderNativeTriggers(s.content, s.language); // Carry the parsed `// tag` so previews route to the same worker the // deployed pipeline would (both `pipeline run --local` and `/pipeline_dev`). pipelineScripts.push(out.tag ? { ...s, tag: out.tag } : s); @@ -420,7 +446,7 @@ export async function buildLocalPipelineGraph(args: { ...(out.partition ? { partition_kind: out.partition.kind } : {}), ...(out.freshness ? { freshness: out.freshness.duration } : {}), ...(out.tag ? { tag: out.tag } : {}), - ...(out.retry ? { retry: out.retry } : {}), + ...(retry ? { retry } : {}), ...(out.data_tests && out.data_tests.length > 0 ? { data_tests: out.data_tests } : {}), ...(out.column_lineage && out.column_lineage.length > 0 ? { column_lineage: out.column_lineage } @@ -466,6 +492,7 @@ export async function buildLocalPipelineGraph(args: { }); } } + const existingNativeTriggers = new Set(); for (const t of out.triggers ?? []) { if (t.kind === "asset") { const at = t as { kind: "asset"; asset_kind: string; path: string }; @@ -481,6 +508,7 @@ export async function buildLocalPipelineGraph(args: { runnable_path: s.path, }); } else { + existingNativeTriggers.add(t.kind); triggers.push({ trigger_kind: t.kind, runnable_kind: "script", @@ -488,6 +516,14 @@ export async function buildLocalPipelineGraph(args: { }); } } + for (const kind of nativeTriggers) { + if (existingNativeTriggers.has(kind)) continue; + triggers.push({ + trigger_kind: kind, + runnable_kind: "script", + runnable_path: s.path, + }); + } } return { diff --git a/cli/test/pipeline_local_graph_unit.test.ts b/cli/test/pipeline_local_graph_unit.test.ts index 6b192cb317..1368e14504 100644 --- a/cli/test/pipeline_local_graph_unit.test.ts +++ b/cli/test/pipeline_local_graph_unit.test.ts @@ -67,11 +67,32 @@ test("native triggers surface as trigger rows (no deploy needed)", async () => { { "ingest.bun.ts": `// pipeline\n// on data_upload\nimport * as wmill from "windmill-client"\nexport async function main() {}\n`, + "upload_trigger.duckdb.sql": `-- pipeline\n-- on data_upload\nSELECT 1;\n`, }, async (root, folder) => { const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); const native = graph.triggers.filter((t) => t.trigger_kind !== "asset"); - expect(native.map((t) => t.trigger_kind)).toContain("data_upload"); + expect( + native + .filter((t) => t.trigger_kind === "data_upload") + .map((t) => t.runnable_path) + .sort(), + ).toEqual(["f/mypipe/ingest", "f/mypipe/upload_trigger"]); + }, + ); +}); + +test("retry delay metadata strips the optional `delay=` prefix", async () => { + await withFolder( + { + "retry.duckdb.sql": `-- pipeline\n-- retry 2 delay=10s\nSELECT 1;\n`, + }, + async (root, folder) => { + const { graph } = await buildLocalPipelineGraph({ root, folder, defaultTs: "bun" }); + expect(graph.runnables.find((r) => r.path === "f/mypipe/retry")?.retry).toEqual({ + count: 2, + delay: "10s", + }); }, ); }); From 789450731b0a3c8dffa336f7bfc3f3de528c09fb Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 2 Jul 2026 12:51:36 +0200 Subject: [PATCH 44/56] fix(triggers): retry transient websocket connect failures before disabling (#9887) A websocket trigger is permanently disabled (with a critical alert) when a single connect attempt in get_consumer fails. Gateway endpoints fronted by edge proxies (e.g. Discord behind Cloudflare) sporadically answer the upgrade handshake with a transient 502/520, so a long-lived trigger that reconnects frequently eventually catches one and dies until a human re-enables it. Retry transient failures (HTTP 5xx/429 handshake responses and IO errors) up to 5 attempts with exponential backoff before surfacing the error, and report retry progress through the trigger's error status. Permanent-looking errors (bad URL, other 4xx, protocol/TLS mismatch) still disable immediately. Co-authored-by: Claude Fable 5 --- .../src/listener.rs | 71 +++++++- .../tests/connect_retry.rs | 153 ++++++++++++++++++ 2 files changed, 216 insertions(+), 8 deletions(-) create mode 100644 backend/windmill-trigger-websocket/tests/connect_retry.rs diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index d46faf1b1a..e34fe6e4f6 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -11,7 +11,7 @@ use serde::Deserialize; use serde_json::value::RawValue; use std::{borrow::Cow, collections::HashMap, sync::Arc}; use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{tungstenite, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use windmill_common::{ error::{to_anyhow, Error, Result}, jobs::JobTriggerKind, @@ -22,13 +22,30 @@ use windmill_common::{ }; use windmill_queue::PushArgsOwned; use windmill_trigger::filter::{check_filters, Filter}; -use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::listener::{update_rw_lock, ListeningTrigger}; use windmill_trigger::trigger_helpers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, }; use windmill_trigger::Listener; +const MAX_CONNECT_ATTEMPTS: u32 = 5; + +/// Whether a failed WebSocket connect is worth retrying: network-level IO +/// errors and HTTP 5xx/429 handshake responses are typically transient (edge +/// proxies like Cloudflare return 502/520 sporadically), while other errors +/// (bad URL, protocol mismatch, other 4xx) point at configuration and would +/// fail identically on every attempt. +fn is_transient_connect_error(err: &tungstenite::Error) -> bool { + match err { + tungstenite::Error::Io(_) => true, + tungstenite::Error::Http(resp) => { + resp.status().is_server_error() || resp.status().as_u16() == 429 + } + _ => false, + } +} + async fn send_initial_messages( listening_trigger: &ListeningTrigger, writer: &mut SplitSink>, Message>, @@ -176,12 +193,50 @@ impl Listener for WebsocketTrigger { validate_websocket_url_for_ssrf(&connect_url).await?; - let connection = connect_async_with_proxy(&*connect_url) - .await - .map(|conn| Some(conn)) - .map_err(|err| to_anyhow(err).into()); - - connection + // Gateway endpoints are often fronted by an edge proxy (e.g. Cloudflare) + // that sporadically answers the upgrade request with a transient 5xx + // instead of `101 Switching Protocols`, and a `get_consumer` error + // disables the trigger until a human re-enables it — so retry transient + // failures with backoff before giving up. The caller runs `loop_ping` + // concurrently so `last_server_ping` stays alive across the sleeps, and + // killpill cancels this future between awaits. + let mut attempt = 0; + loop { + attempt += 1; + match connect_async_with_proxy(&*connect_url).await { + Ok(conn) => return Ok(Some(conn)), + // Only retry in trigger mode: a failed connect there disables the + // trigger until a human re-enables it, while capture mode is an + // interactive test where instant feedback beats resilience (an + // `Io` error can also be a permanent misconfiguration, e.g. a + // typo'd host, which should surface immediately when iterating). + Err(err) + if listening_trigger.trigger_mode + && attempt < MAX_CONNECT_ATTEMPTS + && is_transient_connect_error(&err) => + { + let delay_secs = 1u64 << attempt; + tracing::warn!( + "Transient error connecting to WebSocket for trigger {} (attempt {}/{}), retrying in {}s: {}", + listening_trigger.path, + attempt, + MAX_CONNECT_ATTEMPTS, + delay_secs, + err + ); + update_rw_lock( + err_message.clone(), + Some(format!( + "Connection attempt {}/{} failed ({}), retrying in {}s...", + attempt, MAX_CONNECT_ATTEMPTS, err, delay_secs + )), + ) + .await; + tokio::time::sleep(std::time::Duration::from_secs(delay_secs)).await; + } + Err(err) => return Err(to_anyhow(err).into()), + } + } } async fn consume( &self, diff --git a/backend/windmill-trigger-websocket/tests/connect_retry.rs b/backend/windmill-trigger-websocket/tests/connect_retry.rs new file mode 100644 index 0000000000..2d21512692 --- /dev/null +++ b/backend/windmill-trigger-websocket/tests/connect_retry.rs @@ -0,0 +1,153 @@ +//! End-to-end tests for the websocket trigger connect-retry behavior: a mock +//! TCP server rejects the websocket upgrade with a configurable HTTP status a +//! number of times before completing a real handshake, and the tests assert +//! which failures `get_consumer` retries. +//! +//! This lives in an integration-test binary (own process) because it sets +//! ALLOW_PRIVATE_WEBSOCKET_URLS — the mock server listens on 127.0.0.1, which +//! the SSRF check blocks — and that process-global env var must not leak into +//! the crate's unit tests, which assert loopback URLs are rejected. + +use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, +}; + +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + sync::{broadcast, RwLock}, +}; +use windmill_trigger::{listener::ListeningTrigger, Listener}; +use windmill_trigger_websocket::{ + WebsocketConfig, WebsocketTrigger, ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, +}; + +/// Mock server: rejects the first `failures` upgrade requests with +/// `status_line` and closes, then completes real websocket handshakes and +/// parks the connection open. Returns the bound address and the +/// connection-attempt counter. +async fn mock_ws_server( + failures: u32, + status_line: &'static str, +) -> (std::net::SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let attempts = Arc::new(AtomicU32::new(0)); + let served = attempts.clone(); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let n = served.fetch_add(1, Ordering::SeqCst); + if n < failures { + // Drain the request head, then reject the upgrade. + let mut reader = BufReader::new(&mut socket); + let mut line = String::new(); + loop { + line.clear(); + let read = reader.read_line(&mut line).await.unwrap_or(0); + if read == 0 || line == "\r\n" { + break; + } + } + socket + .write_all( + format!("HTTP/1.1 {status_line}\r\nContent-Length: 0\r\n\r\n").as_bytes(), + ) + .await + .ok(); + } else if let Ok(ws) = tokio_tungstenite::accept_async(socket).await { + tokio::spawn(async move { + let _open = ws; + std::future::pending::<()>().await + }); + } + } + }); + (addr, attempts) +} + +fn trigger(url: String, trigger_mode: bool) -> ListeningTrigger { + ListeningTrigger { + path: "f/test/ws".to_string(), + is_flow: false, + workspace_id: "test".to_string(), + edited_by: "test".to_string(), + permissioned_as: "u/test".to_string(), + trigger_config: WebsocketConfig { + url, + filters: vec![], + filter_logic: "and".to_string(), + initial_messages: None, + url_runnable_args: None, + can_return_message: false, + can_return_error_result: false, + heartbeat: None, + }, + script_path: "f/test/script".to_string(), + trigger_mode, + error_handling: None, + suspended_mode: false, + } +} + +async fn get_consumer_result( + lt: &ListeningTrigger, + err_message: Arc>>, +) -> windmill_common::error::Result::Consumer>> { + // The static-URL path of `get_consumer` never touches the DB; a lazy pool + // satisfies the signature without a running postgres. + let db: windmill_common::DB = + sqlx::Pool::connect_lazy("postgres://unused:unused@127.0.0.1:1/unused").unwrap(); + let (_killpill_tx, killpill_rx) = broadcast::channel::<()>(1); + WebsocketTrigger + .get_consumer(&db, lt, err_message, killpill_rx) + .await +} + +#[tokio::test] +async fn transient_502s_are_retried_until_the_upgrade_succeeds() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(2, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), true); + let err_message = Arc::new(RwLock::new(None)); + + let consumer = get_consumer_result(<, err_message.clone()) + .await + .expect("connect should succeed after retries"); + + assert!(consumer.is_some(), "expected an established connection"); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + // Retry progress was reported through the shared status lock. + let status = err_message.read().await.clone().unwrap(); + assert!(status.contains("attempt 2/5"), "got status: {status}"); + assert!(status.contains("502"), "got status: {status}"); +} + +#[tokio::test] +async fn non_transient_http_errors_fail_on_the_first_attempt() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "404 Not Found").await; + let lt = trigger(format!("ws://{addr}"), true); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("a 404 upgrade response should not be retried"); + + assert!(err.to_string().contains("404"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn capture_mode_fails_fast_even_on_transient_errors() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), false); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("capture mode should surface the first failure immediately"); + + assert!(err.to_string().contains("502"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} From 5af91a677cad88faccba702e3556fc4fb7b6e640 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 2 Jul 2026 16:32:01 +0200 Subject: [PATCH 45/56] fix: stale AI chat context picker after workspace item changes (#9893) * fix: revalidate workspace items cache on context picker open The chat context picker (and workspace drill pickers) read from a module-level cache that was only invalidated from two editor save paths, so items created or deleted anywhere else stayed stale until a full page reload. Make the loader do real stale-while-revalidate: cached items render instantly and each workspace+kind is re-fetched once per picker mount, keeping the state reference stable when nothing changed. Co-Authored-By: Claude Fable 5 * fix: retry failed picker revalidation and handle its rejection Mark a workspace+kind as revalidated only after the fetch succeeds, so a failed revalidation is retried on the next ensureLoaded call instead of stranding stale data for the rest of the mount. Catch the rejection (callers fire-and-forget) and log it. Also dedupe the stale-while-revalidate rationale to its canonical comment site in workspacePicker.ts. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../components/workspaceItemsLoader.svelte.ts | 40 ++++++++++++++++--- .../src/lib/components/workspacePicker.ts | 14 ++++--- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/workspaceItemsLoader.svelte.ts b/frontend/src/lib/components/workspaceItemsLoader.svelte.ts index 45d64bd01b..4ac65f805b 100644 --- a/frontend/src/lib/components/workspaceItemsLoader.svelte.ts +++ b/frontend/src/lib/components/workspaceItemsLoader.svelte.ts @@ -21,13 +21,23 @@ import { * Both getters are read inside the returned closures so changing * workspace or kinds after mount Just Works. */ +function itemsEqual(a: WorkspaceItem[] | undefined, b: WorkspaceItem[]): boolean { + if (!a || a.length !== b.length) return false + return a.every( + (item, i) => + item.path === b[i].path && + item.summary === b[i].summary && + item.kind === b[i].kind && + item.raw_app === b[i].raw_app + ) +} + export function useWorkspaceItemsLoader( workspace: () => string | undefined, kinds: () => readonly WorkspaceItemKind[] ) { // Seed from the module-level cache so kinds already fetched in this - // session render on the first frame. Re-fetching `ensureLoaded` later - // quietly swaps in fresh data (stale-while-revalidate). + // session render on the first frame. let loaded = $state>>( (() => { const ws = untrack(workspace) @@ -41,18 +51,38 @@ export function useWorkspaceItemsLoader( })() ) let loadingKind = $state>>({}) + // Workspace+kind pairs this loader instance has refreshed — at most one + // background re-fetch per kind per mount. Marked on success only, so a + // failed fetch is retried by the next call instead of stranding stale data. + const revalidated = new Set() async function ensureLoaded(kind: WorkspaceItemKind) { const ws = workspace() if (!ws) return + const revalidateKey = `${ws}:${kind}` // `loaded[kind]` read inside `untrack` so callers wiring this into // a reactive context (DrillPicker's onFilterChange effect) don't // subscribe to a signal `ensureLoaded` itself writes — that would // re-fire the effect on every assignment and busy-loop. - if (!untrack(() => loaded[kind])) loadingKind[kind] = true + const hasCached = !!untrack(() => loaded[kind]) + if (hasCached && revalidated.has(revalidateKey)) return + if (!hasCached) loadingKind[kind] = true try { - const items = await loadKind(ws, kind) - loaded[kind] = items + const items = await loadKind(ws, kind, { revalidate: true }) + revalidated.add(revalidateKey) + // Keep the reference stable when nothing changed so an open picker's + // derived tree isn't rebuilt under the user on every revalidation. + if ( + !itemsEqual( + untrack(() => loaded[kind]), + items + ) + ) + loaded[kind] = items + } catch (e) { + // Callers fire-and-forget; surface the failure without an + // unhandled rejection. Cached items (if any) keep rendering. + console.error(`Failed to load workspace ${kind}s`, e) } finally { loadingKind[kind] = false } diff --git a/frontend/src/lib/components/workspacePicker.ts b/frontend/src/lib/components/workspacePicker.ts index ba2985ecbb..92ea48af20 100644 --- a/frontend/src/lib/components/workspacePicker.ts +++ b/frontend/src/lib/components/workspacePicker.ts @@ -44,9 +44,10 @@ type WorkspaceCache = { } /** Module-level session cache. Persists across picker mounts within a single - * page session. NOT invalidated automatically — call `invalidate()` after - * creating/deleting an item if the picker may be opened again before a full - * reload. */ + * page session so cached kinds render on the first frame. Pickers revalidate + * once per mount via `loadKind(..., { revalidate: true })`, so stale entries + * self-heal on the next open; call `invalidate()` after creating/deleting an + * item when even a brief flash of the stale list must be avoided. */ const cache = new Map() const inflight = new Map>() /** Bumped by `invalidate()`. Each in-flight `loadKind` captures the version @@ -87,11 +88,14 @@ export function invalidate(workspace: string, kind?: WorkspaceItemKind) { export async function loadKind( workspace: string, - kind: WorkspaceItemKind + kind: WorkspaceItemKind, + opts?: { revalidate?: boolean } ): Promise { const existing = cache.get(workspace)?.[kind] - if (existing) return existing + if (existing && !opts?.revalidate) return existing const key = cacheKey(workspace, kind) + // An in-flight fetch is already hitting the network, so it satisfies a + // revalidate request too. const flying = inflight.get(key) if (flying) return flying From 37bb57474e8336823bb31527f2a708ef41cd39c4 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 2 Jul 2026 23:18:49 +0200 Subject: [PATCH 46/56] fix: make SMTP username and password optional in frontend validation (#9895) --- .../src/lib/components/instanceSettings/SmtpSettings.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte index c297b1a32f..f7607c5d6c 100644 --- a/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte +++ b/frontend/src/lib/components/instanceSettings/SmtpSettings.svelte @@ -5,10 +5,6 @@ smtpSettings.smtp_host && smtpSettings.smtp_host.trim() !== '' && smtpSettings.smtp_port && - smtpSettings.smtp_username && - smtpSettings.smtp_username.trim() !== '' && - smtpSettings.smtp_password && - smtpSettings.smtp_password.trim() !== '' && smtpSettings.smtp_from && smtpSettings.smtp_from.trim() !== '' ) From 186ac4933b79aed57fce23ebcf3b525fcfd1c474 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 2 Jul 2026 23:30:28 +0200 Subject: [PATCH 47/56] fix(frontend): home New submenus fall back below, hugging the right edge (#9894) * fix(frontend): home New submenus fall back below hugging the right edge Co-Authored-By: Claude Fable 5 * fix(frontend): re-hug submenu on window resize even without a melt reposition Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../components/home/CreateActionsMenu.svelte | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte index 146e390419..4d8adb88e9 100644 --- a/frontend/src/lib/components/home/CreateActionsMenu.svelte +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -243,7 +243,9 @@ forceVisible: true }) - // per-row fan-out submenus, anchored toward the page center (popover hugs the right edge) + // per-row fan-out submenus: open to the right when there's room. The popover hugs + // the right viewport edge on most screens, so the default flip (right → left) would + // cover the doc panel — fall back below the trigger row instead. const { elements: { subTrigger: wacSubTrigger, subMenu: wacSubMenu }, states: { subOpen: wacSubOpen } @@ -251,7 +253,7 @@ positioning: { placement: 'right-start', gutter: 4, - flip: true, + flip: { fallbackPlacements: ['bottom-end', 'bottom-start', 'top-end', 'top-start'] }, fitViewport: true, overflowPadding: 8 } @@ -263,12 +265,43 @@ positioning: { placement: 'right-end', gutter: 4, - flip: true, + flip: { fallbackPlacements: ['bottom-end', 'bottom-start', 'top-end', 'top-start'] }, fitViewport: true, overflowPadding: 8 } }) + // When a fan-out submenu falls below its row ('bottom'/'top' data-side), melt aligns + // it to the row's right edge, but free viewport space may remain beside the popover — + // slide it right until it hugs the viewport edge. Melt repositions on scroll/resize + // via style/data-side writes, so re-apply on attribute mutations. + function hugViewportRight(node: HTMLElement) { + let delta = 0 + const apply = () => { + const side = node.getAttribute('data-side') + const baseRight = node.getBoundingClientRect().right - delta + const next = + side === 'bottom' || side === 'top' + ? Math.max(0, document.documentElement.clientWidth - 8 - baseRight) + : 0 + if (Math.abs(next - delta) < 0.5) return + delta = next + node.style.transform = next ? `translateX(${next}px)` : '' + } + const observer = new MutationObserver(apply) + observer.observe(node, { attributeFilter: ['style', 'data-side'] }) + // melt only rewrites style when the reposition moves the submenu — a resize that + // changes clientWidth without moving it (scrollbar appearing, zoom) needs a direct hook + window.addEventListener('resize', apply) + apply() + return { + destroy: () => { + observer.disconnect() + window.removeEventListener('resize', apply) + } + } + } + // attach the menu trigger to the design-system
- - - -
- {#if currentName} -

Current name
{currentName}

- {/if} - -
- - {#snippet actions()} - {/snippet} - + + From d15033cde6a474b548ebbaf18ff02223fc21f701 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 2 Jul 2026 21:57:26 +0000 Subject: [PATCH 50/56] fix: invalidate bun bundle cache on transitive relative-import changes (#9891) * fix: invalidate bun bundle cache on transitive relative-import changes Co-Authored-By: Claude Fable 5 * fix: do not memoize transient fetch errors in bundle-key import cache Co-Authored-By: Claude Fable 5 * fix: use regular comment on lazy_static block (deny unused_doc_comments) Co-Authored-By: Claude Fable 5 * fix: align bundle-key import version selection with loader content endpoint Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...44cfa928469ce3c1a53ec273ecc997f6e61f8.json | 23 +++ ...1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json | 23 --- backend/src/main.rs | 4 + backend/tests/bun_jobs.rs | 121 ++++++++++++++++ backend/windmill-common/src/lib.rs | 44 ++++++ backend/windmill-worker/src/bun_executor.rs | 131 +++++++++++++++--- 6 files changed, 301 insertions(+), 45 deletions(-) create mode 100644 backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json delete mode 100644 backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json diff --git a/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json new file mode 100644 index 0000000000..e171d44af5 --- /dev/null +++ b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8" +} diff --git a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json b/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json deleted file mode 100644 index f65ea6d2d4..0000000000 --- a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b" -} diff --git a/backend/src/main.rs b/backend/src/main.rs index 94b8e2f4f6..0d7bef4d70 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1708,6 +1708,10 @@ async fn process_notify_event( match *source_type { "script" => { windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + // Bundle-cache key resolution for imported scripts; evicted + // together with the content-side caches below so key and + // inlined content flip to the new version in the same window. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&key); // Evict the relative-import latest-hash cache so a redeployed // imported script flips the content cache to its new version // across all replicas within a poll interval (see #6769). Keyed diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index d51e51f58b..fc766253bc 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,3 +1,4 @@ +use futures::StreamExt; use sqlx::postgres::Postgres; use sqlx::Pool; use uuid::Uuid; @@ -818,6 +819,126 @@ export function main() { Ok(()) } +// ============================================================================ +// Bundle cache invalidation on transitive relative-import change +// ============================================================================ + +async fn insert_deployed_bun_script(db: &Pool, path: &str, hash: i64, content: &str) { + // What gen_bun_lockfile stores for a script with no npm dependencies; a + // bare '' lock fails split_lockfile when the script is run directly. + const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + // Runtime query to avoid touching the sqlx offline cache. + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"properties\":{},\"required\":[],\"type\":\"object\"}', '', '', $2, $3, 'bun', $4)", + ) + .bind(content) + .bind(path) + .bind(hash) + .bind(EMPTY_BUN_LOCK) + .execute(db) + .await + .unwrap(); +} + +fn run_main_script_job(hash: i64) -> RunJob { + RunJob::from(JobPayload::ScriptHash { + path: "f/stale_bundle/main_script".to_string(), + hash: windmill_common::scripts::ScriptHash(hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }) +} + +/// Editing a script that a runnable imports only TRANSITIVELY (main -> mid -> +/// leaf) must invalidate the runnable's cached bundle: the leaf's code is +/// inlined in the bundle, so the cache key has to cover the whole closure, not +/// just direct imports. +#[sqlx::test(fixtures("base"))] +async fn test_bun_transitive_import_change_rebundles(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Hashes/paths unique across this test binary: the script/hash caches are + // process-global while parallel tests each run in their own DB. + const LEAF_V1: i64 = 41230001; + const MID: i64 = 41230002; + const MAIN: i64 = 41230003; + const LEAF_V2: i64 = 41230004; + + insert_deployed_bun_script( + &db, + "f/stale_bundle/leaf", + LEAF_V1, + r#"export function leafValue() { return "V1_FROM_LEAF"; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/mid", + MID, + r#"import { leafValue } from "./leaf"; +export function midValue() { return `M(${leafValue()})`; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/main_script", + MAIN, + r#"import { midValue } from "./mid"; +export function main() { return midValue(); }"#, + ) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + in_test_worker( + &db, + async move { + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V1_FROM_LEAF)")); + + // Deploy a new version of ONLY the leaf; main_script and mid keep + // their hash, content, and lock byte-identical. + insert_deployed_bun_script( + &db2, + "f/stale_bundle/leaf", + LEAF_V2, + r#"export function leafValue() { return "V2_FROM_LEAF"; }"#, + ) + .await; + + // Tests don't run the notify_event poll loop, so replay what its + // `notify_runnable_version_change` handler (main.rs) does on deploy: + // evict the leaf's latest-hash cache entries. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&( + "test-workspace".to_string(), + "f/stale_bundle/leaf".to_string(), + )); + windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE + .remove(&format!("test-workspace:f/stale_bundle/leaf")); + + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V2_FROM_LEAF)")); + }, + port, + ) + .await; + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index aaead77925..baa3e4b500 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -268,6 +268,10 @@ lazy_static::lazy_static! { pub static ref INSTANCE_NAME: String = rd_string(5); pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + // Latest non-archived version per (workspace, path) for bundle cache keying — + // looser predicate than DEPLOYED_SCRIPT_HASH_CACHE (no lock requirement), so + // the two must not share entries. See get_latest_script_hash_for_import_cached. + pub static ref IMPORTED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref DYNAMIC_INPUT_CACHE: Cache> = Cache::new(1000); pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); @@ -1602,6 +1606,46 @@ pub async fn get_latest_script_hash<'e, E: sqlx::PgExecutor<'e>>( return Ok(hash); } +/// Latest non-archived hash for an imported `path`, for bundle cache keying. +/// MUST select the same row as the bundler's content endpoint +/// (`raw_script_by_path_internal`: `archived = false ORDER BY created_at DESC`, +/// no lock predicate) — a stricter filter here would let the key point at an +/// older version than the content that gets inlined. Cached with the same +/// freshness contract as that endpoint's `RAW_SCRIPT_LATEST_HASH_CACHE`: +/// evicted by `notify_runnable_version_change` events, 60s TTL fallback. +pub async fn get_latest_script_hash_for_import_cached( + db: &DB, + w_id: &str, + script_path: &str, +) -> error::Result> { + let use_cache = !DEPLOYED_SCRIPT_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + let cache_key = (w_id.to_string(), script_path.to_string()); + if use_cache { + if let Some(cached) = IMPORTED_SCRIPT_HASH_CACHE.get(&cache_key) { + if cached.expires_at > std::time::Instant::now() { + return Ok(Some(cached.id)); + } + } + } + let hash = sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await?; + if let (true, Some(hash)) = (use_cache, hash) { + IMPORTED_SCRIPT_HASH_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: hash, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + } + Ok(hash) +} + pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>( db_authed: Option>>, db: E, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1e4f4aafd6..f6b093f766 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,6 +1,11 @@ #[cfg(feature = "deno_core")] use std::time::Instant; -use std::{collections::HashMap, fs, process::Stdio}; +use std::{ + collections::{HashMap, HashSet}, + fs, + process::Stdio, + sync::Arc, +}; use base64::Engine; use itertools::Itertools; @@ -27,9 +32,10 @@ use crate::{ NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ + cache, client::AuthedClient, jobs::JobKind, - scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, + scripts::{id_to_codebase_info, CodebaseInfo, ScriptHash, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, }; @@ -44,7 +50,6 @@ use tokio::io::AsyncReadExt; use windmill_common::{ error::{self, Result}, - get_latest_hash_for_path, worker::{write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -1242,16 +1247,101 @@ pub fn ensure_bundle_output_exists(bundle_path: &str) -> Result<()> { pub const BUN_BUNDLE_OBJECT_STORE_PREFIX: &str = "bun_bundle/"; -async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> Result { - let script_hash = get_latest_hash_for_path(db, w_id, script_path, false).await?; - let last_updated_at = sqlx::query_scalar!( - "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - w_id, - script_hash.0 .0 +// A script version's relative-import list never changes (content is immutable +// per hash), so parses are memoized without any invalidation. +lazy_static::lazy_static! { + static ref RELATIVE_IMPORTS_PER_HASH: quick_cache::sync::Cache>> = + quick_cache::sync::Cache::new(1000); +} + +const MAX_TRANSITIVE_IMPORT_PATHS: usize = 256; + +/// `(path, latest hash)` for the whole transitive closure of relative imports +/// of `inner_content` — the set of scripts whose code gets inlined into the +/// bundle, so all of them must key the bundle cache. Resolution goes through +/// `IMPORTED_SCRIPT_HASH_CACHE` (notify-evicted, 60s TTL fallback, same +/// version-selection predicate as the loader's content endpoint) and the +/// per-hash script/parse caches, so steady state costs no DB queries. +async fn collect_transitive_import_versions( + db: &DB, + w_id: &str, + script_path: &str, + inner_content: &str, +) -> Vec<(String, i64)> { + let conn = Connection::from(db.clone()); + let mut queue = crate::worker_lockfiles::extract_relative_imports( + inner_content, + script_path, + &Some(ScriptLang::Bun), ) - .fetch_one(db) - .await?; - Ok(last_updated_at.to_string()) + .unwrap_or_default(); + let mut visited: HashSet = HashSet::new(); + let mut versions: Vec<(String, i64)> = vec![]; + while let Some(path) = queue.pop() { + if !visited.insert(path.clone()) { + continue; + } + if visited.len() > MAX_TRANSITIVE_IMPORT_PATHS { + tracing::warn!( + "transitive relative-import closure of {script_path} exceeds \ + {MAX_TRANSITIVE_IMPORT_PATHS} scripts; bundle cache key covers only the first \ + {MAX_TRANSITIVE_IMPORT_PATHS}" + ); + break; + } + let hash = match windmill_common::get_latest_script_hash_for_import_cached(db, w_id, &path) + .await + { + Ok(Some(hash)) => hash, + // Not a deployed script at this path (deleted, or not a script): + // excluded from the key, matching what the bundler can inline. + Ok(None) => continue, + Err(e) => { + tracing::warn!( + "could not resolve import {path} while computing bundle cache key for \ + {script_path}: {e:#}" + ); + continue; + } + }; + versions.push((path.clone(), hash)); + let imports = match RELATIVE_IMPORTS_PER_HASH.get(&hash) { + Some(imports) => imports, + None => match cache::script::fetch(&conn, ScriptHash(hash)).await { + Ok((data, meta)) => { + let imports = Arc::new(match meta.language { + Some(ScriptLang::Bun) + | Some(ScriptLang::Bunnative) + | Some(ScriptLang::Deno) => { + crate::worker_lockfiles::extract_relative_imports( + &data.code, + &path, + &meta.language, + ) + .unwrap_or_default() + } + _ => vec![], + }); + RELATIVE_IMPORTS_PER_HASH.insert(hash, imports.clone()); + imports + } + // A fetch error is transient, not a property of the (immutable) + // content — memoizing it would drop this subtree from the key + // until worker restart. Skip caching and retry next run. + Err(e) => { + tracing::warn!( + "could not fetch import {path} (hash {hash}) while computing bundle \ + cache key for {script_path}: {e:#}" + ); + Arc::new(vec![]) + } + }, + }; + queue.extend(imports.iter().cloned()); + } + // deterministic key regardless of traversal order + versions.sort(); + versions } pub async fn compute_bundle_local_and_remote_path( @@ -1265,16 +1355,13 @@ pub async fn compute_bundle_local_and_remote_path( let mut input_src = format!("{inner_content}{lock}",); if let Some(db) = db { - let relative_imports = crate::worker_lockfiles::extract_relative_imports( - &inner_content, - script_path, - &Some(ScriptLang::Bun), - ); - for path in relative_imports.unwrap_or_default() { - if let Ok(updated_at) = get_script_import_updated_at(&db, w_id, &path).await { - input_src.push_str(&path); - input_src.push_str(&updated_at.to_string()); - } + // The bundle inlines the whole transitive relative-import closure, so a + // new deployed version of ANY script in it must change the key. + for (path, hash) in + collect_transitive_import_versions(db, w_id, script_path, inner_content).await + { + input_src.push_str(&path); + input_src.push_str(&hash.to_string()); } }; From 7c7d7474cc86a4052272032f281cc4d7a85db37b Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 2 Jul 2026 23:59:02 +0200 Subject: [PATCH 51/56] feat: support workspace forks on cloud using parent workspace limits (#9864) * feat: support workspace forks on cloud using parent workspace limits Co-Authored-By: Claude Opus 4.8 (1M context) * docs: clarify count_paid_seats approximates rather than mirrors billing seats Co-Authored-By: Claude Opus 4.8 (1M context) * fix: non-admin fork UI, attach cap, and fork-count for cloud forks Co-Authored-By: Claude Opus 4.8 (1M context) * fix: cloud fork billing cache on rename, usage display, attach cap edge Co-Authored-By: Claude Opus 4.8 (1M context) * feat: fork count in cloud quotas + fork billing points to parent Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate billing/fork caches on fork deletion for id reuse Co-Authored-By: Claude Opus 4.8 (1M context) * fix: gate fork usage remap on CLOUD_HOSTED, not just the cloud feature Co-Authored-By: Claude Opus 4.8 (1M context) * docs: note cloud feature vs CLOUD_HOSTED gating in backend guide Co-Authored-By: Claude Opus 4.8 (1M context) * fix: reserve fork-cap slots for an attach candidate's whole subtree Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate team-plan cache on delete, raise fork depth cap Co-Authored-By: Claude Opus 4.8 (1M context) * feat: cap fork nesting depth (MAX_FORK_DEPTH, default 5) Co-Authored-By: Claude Opus 4.8 (1M context) * fix: fork count/height robust to cycles and deleted intermediates Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): reset fork button loading state on creation error Co-Authored-By: Claude Opus 4.8 (1M context) * fix: invalidate billing cache for attached fork subtree; helper auth docs Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- ...0b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json | 22 ++ ...bc9b6a776a4941e68d8e2b31904c677493e2d.json | 22 ++ ...eaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json | 22 ++ ...dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json | 22 ++ ...1af05c5a6882f988ba5e0ec1f63828353987b.json | 22 ++ ...adcdd88b137bb892437a3ca67fd716772512b.json | 28 +++ backend/CLAUDE.md | 10 + .../windmill-api-workspaces/src/workspaces.rs | 177 +++++++++++++- .../src/workspaces_extra.rs | 33 ++- backend/windmill-api/openapi.yaml | 3 + backend/windmill-common/src/workspaces.rs | 216 +++++++++++++++++- .../tests/billing_workspace.rs | 211 +++++++++++++++++ backend/windmill-queue/src/jobs.rs | 35 ++- .../components/sessions/SessionForkBar.svelte | 9 +- .../sessions/WorkspaceFamilyPicker.svelte | 13 +- .../components/settings/CloudQuotas.svelte | 4 +- .../components/sidebar/WorkspaceMenu.svelte | 2 +- .../CreateWorkspaceInner.svelte | 39 ++-- .../src/routes/(root)/(logged)/+layout.svelte | 11 +- .../(logged)/workspace_settings/+page.svelte | 11 +- 20 files changed, 860 insertions(+), 52 deletions(-) create mode 100644 backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json create mode 100644 backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json create mode 100644 backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json create mode 100644 backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json create mode 100644 backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json create mode 100644 backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json create mode 100644 backend/windmill-common/tests/billing_workspace.rs diff --git a/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json new file mode 100644 index 0000000000..508eb32ed6 --- /dev/null +++ b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT COALESCE(MAX(depth), 0)::bigint AS \"depth!\" FROM chain\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "depth!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2" +} diff --git a/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json new file mode 100644 index 0000000000..44d9f8f36d --- /dev/null +++ b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS \"height!\" FROM tree\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d" +} diff --git a/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json new file mode 100644 index 0000000000..cb6bb20397 --- /dev/null +++ b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COUNT(DISTINCT id) AS \"count!\" FROM tree WHERE id != $1 AND NOT deleted\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba" +} diff --git a/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json new file mode 100644 index 0000000000..acfa08d0bf --- /dev/null +++ b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT id AS \"id!\" FROM tree WHERE id != $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d" +} diff --git a/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json new file mode 100644 index 0000000000..514024eb8d --- /dev/null +++ b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b" +} diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json new file mode 100644 index 0000000000..1930427b94 --- /dev/null +++ b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "developers!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" +} diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 086e251962..ff5d9b2f22 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -11,3 +11,13 @@ cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh ``` Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. + +## Cloud vs self-hosted gating + +The `cloud` cargo feature is compiled into **all** EE builds, so `#[cfg(feature = "cloud")]` is **not** a "cloud-only" runtime gate — it only means the code is present. The real gate for behavior specific to the managed cloud (app.windmill.dev) is the runtime flag `*CLOUD_HOSTED` (`windmill_common::worker::CLOUD_HOSTED`, from the `CLOUD_HOSTED` env var; note it's loaded from `.env` via `dotenv`, so it won't show in `/proc//environ` — check the running behavior, not the exec env). + +Cloud-only logic must be behind `if *CLOUD_HOSTED { ... }`: feature-gate the helper so it compiles, then **runtime-gate the call**. `#[cfg(feature = "cloud")]` on its own is only sufficient for: +- pure helper/struct definitions (they only run when a gated caller invokes them), +- code already inside an `if *CLOUD_HOSTED { ... }` block, +- handlers that early-return on `!*CLOUD_HOSTED`, +- idempotent no-ops that are harmless off-cloud (e.g. cache invalidation). diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index bb77a50973..171ec94560 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -629,11 +629,13 @@ async fn list_pending_invites( } async fn is_premium( - authed: ApiAuthed, + _authed: ApiAuthed, Extension(_db): Extension, Path(_w_id): Path, ) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; + // Any workspace member (not just admins) may read whether the workspace is on a paid plan: it's a + // single boolean, and the frontend needs it to decide whether to surface premium-gated affordances + // (e.g. forking) to non-admin developers too. The `_authed` extractor still enforces membership. #[cfg(feature = "cloud")] let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) .await? @@ -3797,6 +3799,24 @@ lazy_static::lazy_static! { } }; + // Cloud only: how many forks a premium workspace may have per paid (developer) seat. + pub static ref MAX_FORKS_PER_SEAT: i64 = std::env::var("MAX_FORKS_PER_SEAT") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0) + .unwrap_or(5); + + // How deep a fork chain may nest (root = depth 0, a direct fork = depth 1). A general guardrail + // for all builds, independent of the cloud per-seat cap: deep fork chains are a footgun and no + // real use case needs them. Clamped to [1, 20] so it's always a real limit and can never exceed + // the fork-walk recursion backstop (20) that billing/count resolution uses (a chain deeper than + // the backstop would truncate and mis-resolve its root). + pub static ref MAX_FORK_DEPTH: i64 = std::env::var("MAX_FORK_DEPTH") + .ok() + .and_then(|v| v.parse::().ok()) + .map(|v| v.clamp(1, 20)) + .unwrap_or(5); + } async fn create_workspace_require_superadmin() -> String { @@ -4996,10 +5016,12 @@ async fn create_workspace_fork_branch( Path(w_id): Path, Json(nw): Json, ) -> JsonResult> { + // Pre-check the fork guards before creating any git branch, so we don't leave orphaned branches + // behind when the follow-up create_workspace_fork would be rejected anyway. + enforce_fork_depth(&db, &w_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &w_id).await?; } if *DISABLE_WORKSPACE_FORK { @@ -5171,16 +5193,83 @@ async fn apply_forked_datatable( Ok(()) } +/// Cloud: require the fork/dev's root (billing) workspace be premium; returns the resolved root id. +#[cfg(feature = "cloud")] +async fn require_cloud_fork_premium(db: &DB, parent_workspace_id: &str) -> Result { + let root = + windmill_common::workspaces::get_billing_workspace_id(db, parent_workspace_id).await?; + if !windmill_common::workspaces::get_team_plan_status(db, &root) + .await? + .premium + { + return Err(Error::BadRequest( + "Creating a fork or dev workspace on the cloud requires a paid team plan. Upgrade the workspace first.".to_string(), + )); + } + Ok(root) +} + +/// Cloud: reject if adding `incoming` fork/dev workspaces would push `root`'s family over its per-seat +/// allotment. `incoming` is the number of workspaces the operation adds to the family — 1 for a plain +/// create, but `1 + candidate_subtree` for an attach whose candidate already has child forks. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_count(db: &DB, root: &str, incoming: i64) -> Result<()> { + let seats = windmill_common::workspaces::count_paid_seats(db, root).await?; + let per_seat = *MAX_FORKS_PER_SEAT; + // Any premium workspace has at least one paid seat, so floor the seat count at 1. + let allowed = seats.max(1) * per_seat; + + let existing = windmill_common::workspaces::count_workspace_forks(db, root).await?; + let projected = existing + incoming; + if projected > allowed { + return Err(Error::BadRequest(format!( + "Fork limit reached: this would bring the workspace family to {projected} fork(s), over the cap of {allowed} ({seats} paid seat(s) × {per_seat} per seat). Delete a fork or add seats." + ))); + } + + Ok(()) +} + +/// Cloud-only guard for creating a fork/dev workspace. Forks piggyback on the parent's plan (a fork +/// inherits the root's premium and meters its usage into the root's bill), so forking is limited to +/// premium workspaces and capped at `MAX_FORKS_PER_SEAT` per paid (developer) seat of the root. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<()> { + let root = require_cloud_fork_premium(db, parent_workspace_id).await?; + enforce_cloud_fork_count(db, &root, 1).await +} + +/// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper +/// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new +/// node — 0 for a plain fork, or the candidate's own subtree height for an attach. +async fn enforce_fork_depth( + db: &DB, + parent_workspace_id: &str, + added_subtree_height: i64, +) -> Result<()> { + let parent_depth = + windmill_common::workspaces::fork_chain_depth(db, parent_workspace_id).await?; + // The new node sits one level below the parent; its deepest descendant adds the grafted height. + let resulting_depth = parent_depth + 1 + added_subtree_height; + if resulting_depth > *MAX_FORK_DEPTH { + return Err(Error::BadRequest(format!( + "Fork depth limit reached: forks can be nested at most {} level(s) deep, but this would create a fork at depth {}. Fork from a workspace closer to the root instead.", + *MAX_FORK_DEPTH, resulting_depth + ))); + } + Ok(()) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, Path(parent_workspace_id): Path, Json(nw): Json, ) -> Result { + enforce_fork_depth(&db, &parent_workspace_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &parent_workspace_id).await?; } if nw.is_dev_workspace { @@ -5347,10 +5436,30 @@ async fn attach_dev_workspace( ) -> Result { require_admin(authed.is_admin, &authed.username)?; + // Attaching grafts the candidate (and its own fork subtree) under prod, so enforce the general + // depth limit on the deepest resulting node. + let candidate_height = + windmill_common::workspaces::fork_subtree_height(&db, &req.dev_workspace_id).await?; + enforce_fork_depth(&db, &prod_w_id, candidate_height).await?; + + // Attaching reparents a workspace under prod (one dev per prod, admin-gated) and it then draws + // prod's plan, so hold it to the same premium requirement as creating a fork. Only enforce the + // per-seat count when the attach actually adds a new workspace to the family: re-designating a + // workspace already under this root as its dev doesn't increase the descendant count. + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest( - "Dev workspaces are not available on app.windmill.dev".to_string(), - )); + let root = require_cloud_fork_premium(&db, &prod_w_id).await?; + // Only count against the cap when this attach adds workspaces to the family (candidate not + // already under this root). The candidate may itself have child forks, so reserve slots for its + // whole incoming subtree (the candidate + its descendants), not just one. + if windmill_common::workspaces::get_billing_workspace_id(&db, &req.dev_workspace_id).await? + != root + { + let incoming = + 1 + windmill_common::workspaces::count_workspace_forks(&db, &req.dev_workspace_id) + .await?; + enforce_cloud_fork_count(&db, &root, incoming).await?; + } } let dev_w_id = req.dev_workspace_id; @@ -5468,6 +5577,17 @@ async fn attach_dev_workspace( // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping // so per-workspace job tags route to the prod family immediately rather than after the TTL. windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Same reparent invalidates the billing-workspace mapping so its usage meters to prod at once. The + // candidate can bring its own fork subtree, whose descendants had resolved their (now-stale) root + // to the candidate's old family; invalidate them too so they meter to prod without waiting out the + // 60s TTL. Their immediate fork->parent links don't move, so the tag-routing cache needs no change. + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_billing_workspace_cache(&id); + } + } if req.lock_prod_deploy || req.lock_prod_forking { windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); @@ -6415,6 +6535,17 @@ async fn change_workspace_color( } async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { + // On cloud, a fork's executions meter against its billing root, so report the root's usage here too; + // otherwise the free-execs indicator would show the fork's own (often 0) count while enforcement + // applies the root's shared quota. Gated on `*CLOUD_HOSTED` (not just the `cloud` feature, which is + // compiled into all EE builds): self-hosted doesn't meter usage this way. Off-fork it resolves to + // `w_id` itself anyway. + #[cfg(feature = "cloud")] + let w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id + }; let usage = sqlx::query_scalar!( " SELECT usage.usage FROM usage @@ -8230,6 +8361,10 @@ struct CloudQuotas { apps: QuotaInfo, variables: QuotaInfo, resources: QuotaInfo, + /// Fork/dev workspaces under this workspace's billing root vs the per-seat cap. `limit` is 0 for a + /// non-premium root (forking is premium-only). Family-wide: resolves to the billing root, so it + /// reads the same whether viewed from the root or one of its forks. + forks: QuotaInfo, } async fn get_cloud_quotas( @@ -8310,12 +8445,32 @@ async fn get_cloud_quotas( .await? .unwrap_or(0); + // Fork/dev workspaces vs the per-seat cap, resolved to the billing root. Non-premium roots can't + // fork, so their allowance is 0. + #[cfg(feature = "cloud")] + let forks = { + use windmill_common::workspaces::{ + count_paid_seats, count_workspace_forks, get_billing_workspace_id, get_team_plan_status, + }; + let root = get_billing_workspace_id(&db, &w_id).await?; + let used = count_workspace_forks(&db, &root).await?; + let limit = if get_team_plan_status(&db, &root).await?.premium { + count_paid_seats(&db, &root).await?.max(1) * *MAX_FORKS_PER_SEAT + } else { + 0 + }; + QuotaInfo { used, limit, prunable: 0 } + }; + #[cfg(not(feature = "cloud"))] + let forks = QuotaInfo { used: 0, limit: 0, prunable: 0 }; + Ok(Json(CloudQuotas { scripts: QuotaInfo { used: scripts_used, limit: 5000, prunable: scripts_prunable }, flows: QuotaInfo { used: flows_used, limit: 1000, prunable: flows_prunable }, apps: QuotaInfo { used: apps_used, limit: 1000, prunable: apps_prunable }, variables: QuotaInfo { used: variables_used, limit: 10000, prunable: 0 }, resources: QuotaInfo { used: resources_used, limit: 10000, prunable: 0 }, + forks, })) } diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 47d8c24b92..56abe689d5 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -726,10 +726,13 @@ pub(crate) async fn change_workspace_id( tx.commit().await?; // The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent - // routing cache so jobs route under the renamed root rather than the old (archived) one until the - // 300s TTL would otherwise expire. + // routing cache and their billing-workspace mapping so jobs route + meter under the renamed root + // rather than the old (archived) one, instead of waiting for the caches' TTLs. Deeper descendants + // (fork-of-fork) self-heal via the 60s billing-cache TTL. for child in &reparented_children { windmill_queue::tags::invalidate_fork_parent_cache(child); + #[cfg(feature = "cloud")] + windmill_common::workspaces::invalidate_billing_workspace_cache(child); } // Archive old workspace: disable schedules, cancel remaining jobs, set deleted=true @@ -1017,6 +1020,16 @@ pub(crate) async fn delete_workspace( .await? .flatten(); + // Capture direct child forks before the delete: the FK is ON DELETE SET NULL, so they're about to + // be orphaned (their billing root changes from this workspace's root to themselves). We drop their + // cached mappings after commit alongside the deleted id itself. + let orphaned_children: Vec = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; @@ -1050,6 +1063,22 @@ pub(crate) async fn delete_workspace( windmill_common::workspaces::invalidate_protection_rules_cache(&parent); } + // Workspace ids are reusable after permanent deletion, so drop every cached mapping keyed by the + // deleted id (and any just-orphaned children) — otherwise a recreated id could inherit the gone + // workspace's state within the caches' lifetimes. This covers fork->parent (tag routing) and + // fork->root (billing), plus the premium/team-plan status: TEAM_PLAN_CACHE has no TTL and is only + // evicted by the premium-change NOTIFY, so without this a reused id would keep the old workspace's + // premium indefinitely (free forks/usage). Deeper (grandchild) descendants self-heal via the 60s + // billing TTL. + for id in std::iter::once(&w_id).chain(orphaned_children.iter()) { + windmill_queue::tags::invalidate_fork_parent_cache(id); + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(id); + windmill_common::workspaces::invalidate_team_plan_cache(id); + } + } + Ok(format!("Deleted workspace {}", &w_id)) } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3b90927901..eec76ea009 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6221,12 +6221,15 @@ paths: $ref: "#/components/schemas/QuotaInfo" resources: $ref: "#/components/schemas/QuotaInfo" + forks: + $ref: "#/components/schemas/QuotaInfo" required: - scripts - flows - apps - variables - resources + - forks /w/{workspace}/workspaces/prune_versions: post: diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index cd3693cd92..5593a565ef 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -347,9 +347,215 @@ lazy_static::lazy_static! { pub static ref PUBLIC_APP_RATE_LIMIT_CACHE: Cache, i64)> = Cache::new(1000); } +#[cfg(feature = "cloud")] +lazy_static::lazy_static! { + // Maps a workspace id to its root (billing) ancestor. Value: (root_id, expiry_timestamp). + // Reparenting (attach/detach dev) is rare and self-heals via the 60s TTL, so a brief stale + // mapping only mis-attributes usage for <60s across other instances. + pub static ref BILLING_WORKSPACE_CACHE: Cache = Cache::new(5000); +} + +/// Resolve the "billing" workspace for `w_id`: the root ancestor of the fork/dev chain (the +/// workspace whose plan and usage a fork draws from). Returns `w_id` unchanged for a standalone +/// workspace, an unknown id, or a (malformed) cyclic chain. +/// +/// Unauthenticated metering helper: it only reads the parent chain and returns another workspace id, +/// so callers must already be authorized for `w_id` (or run in trusted server-side code); `w_id` is +/// expected to be a server-side id, not raw user input. +#[cfg(feature = "cloud")] +pub async fn get_billing_workspace_id(db: &crate::DB, w_id: &str) -> Result { + let now = chrono::Utc::now().timestamp(); + if let Some((root, expiry)) = BILLING_WORKSPACE_CACHE.get(w_id) { + if expiry > now { + return Ok(root); + } + } + + // The depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so a + // truncated (root-not-found) result — which would fall back to `w_id` and mis-attribute billing — + // is unreachable for any real hierarchy; only a malformed cycle could hit it. + let root = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT id AS "id!" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1 + "#, + w_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("resolving billing workspace for {w_id}: {e:#}")))? + .unwrap_or_else(|| w_id.to_string()); + + BILLING_WORKSPACE_CACHE.insert(w_id.to_string(), (root.clone(), now + 60)); + Ok(root) +} + +/// Invalidate the billing-workspace mapping for a workspace (call after reparenting it). +#[cfg(feature = "cloud")] +pub fn invalidate_billing_workspace_cache(w_id: &str) { + BILLING_WORKSPACE_CACHE.remove(w_id); +} + +/// Invalidate the cached team-plan (premium/past-due) status for a workspace. `TEAM_PLAN_CACHE` has +/// no TTL — it's only evicted by the premium-change NOTIFY — so call this when a workspace id is +/// permanently deleted, otherwise a reused id could inherit the old workspace's premium status. +#[cfg(feature = "cloud")] +pub fn invalidate_team_plan_cache(w_id: &str) { + TEAM_PLAN_CACHE.remove(w_id); +} + +/// Depth of `w_id` in its fork chain: 0 for a root (no parent), 1 for a direct fork, and so on. Walks +/// the parent chain up to the root. The recursion bound is a cycle-safety backstop set well above the +/// enforced `MAX_FORK_DEPTH`; a (malformed) cyclic chain saturates it and so reads as "too deep", +/// which safely rejects rather than allows. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_chain_depth(db: &crate::DB, w_id: &str) -> Result { + let depth = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT COALESCE(MAX(depth), 0)::bigint AS "depth!" FROM chain + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork depth for {w_id}: {e:#}")))?; + Ok(depth) +} + +/// Height of the fork subtree rooted at `w_id`: 0 when it has no live child forks, 1 with direct +/// children, and so on. Used so that attaching a candidate which already has its own child forks can't +/// push the family past the depth limit. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_subtree_height(db: &crate::DB, w_id: &str) -> Result { + // The `deleted` filter is applied in the outer aggregation (not the recursive step, matching + // count_workspace_forks) so a live descendant under a soft-deleted intermediate is still measured + // at its true depth rather than pruned — otherwise the height could be underestimated and let the + // resulting chain exceed the depth limit. + let height = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS "height!" FROM tree + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork subtree height for {w_id}: {e:#}")))?; + Ok(height) +} + +/// Ids of every fork/dev workspace anywhere under `w_id` (excludes `w_id` itself), including live +/// descendants beneath a soft-deleted intermediate. Used to invalidate per-workspace caches for a +/// whole subtree after its ancestor is reparented. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn list_fork_descendants(db: &crate::DB, w_id: &str) -> Result> { + let ids = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT id AS "id!" FROM tree WHERE id != $1 + "#, + w_id + ) + .fetch_all(db) + .await + .map_err(|e| Error::internal_err(format!("listing fork descendants of {w_id}: {e:#}")))?; + Ok(ids) +} + +/// Count non-deleted fork/dev workspaces anywhere under `root` (excludes `root` itself). +/// +/// Unauthenticated metering helper: it reads workspace hierarchy for any `root` id, so callers must +/// already be authorized for that workspace (or run in trusted server-side code). `root` is expected +/// to be a server-resolved id, never raw user input. +#[cfg(feature = "cloud")] +pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { + // The `deleted` filter is on the outer SELECT (not the recursive step) so that a live sub-fork + // whose intermediate parent was soft-deleted is still counted rather than pruned with it. The + // depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so + // descendants are never silently dropped from the cap count. + let count = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COUNT(DISTINCT id) AS "count!" FROM tree WHERE id != $1 AND NOT deleted + "#, + root + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting forks of {root}: {e:#}")))?; + Ok(count) +} + +/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and +/// service-account members. Reuses billing's author/operator weighting, but counts provisioned +/// members rather than the active-user population billing meters, so it only ever loosens the fork +/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// +/// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + let row = sqlx::query!( + r#"SELECT + COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", + COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS "operators!" + FROM usr WHERE workspace_id = $1"#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; + Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) +} + #[cfg(feature = "cloud")] pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result { - let cached = TEAM_PLAN_CACHE.get(_w_id); + // A fork/dev workspace draws its plan from the root (billing) workspace. Resolve to the root and + // key the cache by it: the premium-change NOTIFY is keyed by the workspace whose premium row + // changed (the root), so keying by root keeps invalidation correct and lets forks share it. + let billing_w_id = get_billing_workspace_id(_db, _w_id).await?; + let cached = TEAM_PLAN_CACHE.get(&billing_w_id); if let Some(cached) = cached { return Ok(cached); } @@ -368,7 +574,7 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result Result Result, id: &str, parent: Option<&str>, deleted: bool) { + sqlx::query( + "INSERT INTO workspace (id, name, owner, parent_workspace_id, deleted) + VALUES ($1, $1, 'test-user', $2, $3)", + ) + .bind(id) + .bind(parent) + .bind(deleted) + .execute(db) + .await + .expect("insert workspace"); + // The resolver caches per id (60s TTL) in a process-global cache shared across tests, so drop + // any stale mapping for this id before the test reads it. + invalidate_billing_workspace_cache(id); +} + +async fn insert_member( + db: &Pool, + w_id: &str, + email: &str, + operator: bool, + disabled: bool, + is_service_account: bool, +) { + // `usr.username` has a `proper_username` check (no `@`), so derive one from the email prefix. + let username = email.split('@').next().unwrap(); + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, operator, disabled, is_service_account, role) + VALUES ($1, $2, $3, false, $4, $5, $6, 'Developer')", + ) + .bind(w_id) + .bind(email) + .bind(username) + .bind(operator) + .bind(disabled) + .bind(is_service_account) + .execute(db) + .await + .expect("insert usr"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_resolves_to_root(db: Pool) { + insert_ws(&db, "bwt-root", None, false).await; + insert_ws(&db, "bwt-fork", Some("bwt-root"), false).await; + insert_ws(&db, "bwt-grandchild", Some("bwt-fork"), false).await; + + assert_eq!( + get_billing_workspace_id(&db, "bwt-root").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-fork").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-grandchild") + .await + .unwrap(), + "bwt-root" + ); + // Unknown / orphaned ids resolve to themselves. + assert_eq!( + get_billing_workspace_id(&db, "bwt-missing").await.unwrap(), + "bwt-missing" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_survives_cycles(db: Pool) { + insert_ws(&db, "bwc-a", None, false).await; + insert_ws(&db, "bwc-b", Some("bwc-a"), false).await; + // Introduce a cycle a -> b -> a; no row has a NULL parent, so resolution falls back to the input + // and the depth guard keeps it from looping forever. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'bwc-b' WHERE id = 'bwc-a'") + .execute(&db) + .await + .unwrap(); + invalidate_billing_workspace_cache("bwc-a"); + invalidate_billing_workspace_cache("bwc-b"); + + assert_eq!( + get_billing_workspace_id(&db, "bwc-a").await.unwrap(), + "bwc-a" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwc-b").await.unwrap(), + "bwc-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn paid_seats_and_fork_count(db: Pool) { + insert_ws(&db, "seat-root", None, false).await; + // 2 developers + 2 operators counted -> ceil(2 + 0.5*2) = 3. + insert_member(&db, "seat-root", "dev1@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; + insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; + // These must NOT count towards seats. + insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; + insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; + + assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + + insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; + insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; + // A deleted fork itself is not counted... + insert_ws(&db, "seat-fork-deleted", Some("seat-root"), true).await; + // ...but a live sub-fork under it still is (deleted filter is on the outer SELECT, not the walk). + insert_ws(&db, "seat-deleted-child", Some("seat-fork-deleted"), false).await; + // A grandchild fork still counts. + insert_ws(&db, "seat-fork1-child", Some("seat-fork1"), false).await; + + // Live: fork1, fork2, fork1-child, deleted-child -> 4 (seat-fork-deleted excluded). + assert_eq!(count_workspace_forks(&db, "seat-root").await.unwrap(), 4); + // A standalone workspace has no forks. + assert_eq!(count_workspace_forks(&db, "seat-fork2").await.unwrap(), 0); + + // list_fork_descendants returns every descendant id (deleted included, for cache invalidation): + // fork1, fork2, fork-deleted, deleted-child, fork1-child -> 5. + let mut descendants = list_fork_descendants(&db, "seat-root").await.unwrap(); + descendants.sort(); + assert_eq!( + descendants, + vec![ + "seat-deleted-child", + "seat-fork-deleted", + "seat-fork1", + "seat-fork1-child", + "seat-fork2", + ] + ); + assert!(list_fork_descendants(&db, "seat-fork2") + .await + .unwrap() + .is_empty()); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_cache_invalidation_reflects_reparent(db: Pool) { + insert_ws(&db, "inv-root-a", None, false).await; + insert_ws(&db, "inv-root-b", None, false).await; + insert_ws(&db, "inv-fork", Some("inv-root-a"), false).await; + + // Resolve + cache: fork -> root-a. + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // Reparent in the DB, as delete+recreate-under-another-root (or attach) would. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'inv-root-b' WHERE id = 'inv-fork'") + .execute(&db) + .await + .unwrap(); + + // The cached mapping survives until invalidated (this is the staleness the delete/attach/rename + // paths must clear). + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // After invalidation (what delete_workspace / attach_dev_workspace / change_workspace_id now call), + // it re-resolves to the new root. + invalidate_billing_workspace_cache("inv-fork"); + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn fork_depth_and_subtree_height(db: Pool) { + // Chain: root -> f1 -> f2 -> f3 + insert_ws(&db, "fd-root", None, false).await; + insert_ws(&db, "fd-f1", Some("fd-root"), false).await; + insert_ws(&db, "fd-f2", Some("fd-f1"), false).await; + insert_ws(&db, "fd-f3", Some("fd-f2"), false).await; + + // Depth walks up to the root: root = 0, each fork adds one. + assert_eq!(fork_chain_depth(&db, "fd-root").await.unwrap(), 0); + assert_eq!(fork_chain_depth(&db, "fd-f1").await.unwrap(), 1); + assert_eq!(fork_chain_depth(&db, "fd-f3").await.unwrap(), 3); + // An unknown id has no chain, so depth 0 (treated as a root by the guard). + assert_eq!(fork_chain_depth(&db, "fd-missing").await.unwrap(), 0); + + // Height walks down: the deepest live descendant below the node. + assert_eq!(fork_subtree_height(&db, "fd-root").await.unwrap(), 3); + assert_eq!(fork_subtree_height(&db, "fd-f2").await.unwrap(), 1); + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // A deleted leaf doesn't add to the height. + insert_ws(&db, "fd-f3-del", Some("fd-f3"), true).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // ...but a LIVE descendant below a soft-deleted intermediate still counts at its true depth + // (the walk traverses through the deleted node; only the aggregation filters deleted). + insert_ws(&db, "fd-f3-live-gc", Some("fd-f3-del"), false).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 2); +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index c92cea9ef3..99f33da3bf 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1914,22 +1914,31 @@ fn apply_completed_job_cloud_usage( tokio::task::spawn(async move { let additional_usage = _duration / 1000; let result = tokio::time::timeout(std::time::Duration::from_secs(10), async move { + // Fork/dev execution-seconds meter against the root (billing) workspace; resolves to + // `w_id` itself off-fork. + let billing_w_id = + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id) + .await + .unwrap_or_else(|e| { + tracing::error!("Failed to resolve billing workspace for {w_id}: {e:#}"); + w_id.clone() + }); // Update workspace usage let workspace_result = sqlx::query!( "INSERT INTO usage (id, is_workspace, month_, usage) VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2) ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage", - &w_id, + &billing_w_id, additional_usage as i32 ) .execute(&db) .await; if let Err(e) = workspace_result { - tracing::error!("Failed to update workspace usage for {}: {:#}", w_id, e); + tracing::error!("Failed to update workspace usage for {}: {:#}", billing_w_id, e); } - match windmill_common::workspaces::get_team_plan_status(&db, &w_id).await { + match windmill_common::workspaces::get_team_plan_status(&db, &billing_w_id).await { Ok(team_plan_status) => { // Update user usage for non-premium workspaces if !team_plan_status.premium { @@ -5124,8 +5133,13 @@ async fn push_inner<'c, 'd>( ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { #[cfg(feature = "cloud")] if *CLOUD_HOSTED { + // A fork/dev workspace draws its plan and usage from the root (billing) workspace, so its + // executions are metered against the parent's quota/bill. Resolves to `workspace_id` itself + // for a standalone workspace (no behavior change off-fork). + let billing_w_id = + windmill_common::workspaces::get_billing_workspace_id(db, workspace_id).await?; let team_plan_status = - windmill_common::workspaces::get_team_plan_status(db, workspace_id).await?; + windmill_common::workspaces::get_team_plan_status(db, &billing_w_id).await?; // we track only non flow steps let (workspace_usage, user_usage) = if !matches!( job_payload, @@ -5134,12 +5148,12 @@ async fn push_inner<'c, 'd>( // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces let (current_workspace_usage, current_user_usage) = - check_usage_limits(db, workspace_id, email, !team_plan_status.premium).await?; + check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; // Spawn async task to update usage counters in the background increment_usage_async( db.clone(), - workspace_id.to_string(), + billing_w_id.clone(), if !team_plan_status.premium { Some(email.to_string()) } else { @@ -5242,7 +5256,7 @@ async fn push_inner<'c, 'd>( WHERE is_workspace IS TRUE AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND id = $1", - workspace_id + billing_w_id ) .fetch_optional(db) .await? @@ -5269,6 +5283,13 @@ async fn push_inner<'c, 'd>( ))); } + // These two burst guards intentionally stay keyed to `workspace_id`, not the + // billing root: the shared caps are the monthly usage above (metered to the + // root) and the per-user in-queue/concurrent guards further up (keyed by email, + // global across the whole family). Keying these to the root would count the + // root's own queue rather than this workspace's load or the family's; a true + // family-shared burst cap would need a family-wide subquery, not worth the + // hot-path cost for this downgrade-only soft guard. let in_queue_workspace = sqlx::query_scalar!( "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", workspace_id diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte index 982dc68c0f..a0aafee456 100644 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ b/frontend/src/lib/components/sessions/SessionForkBar.svelte @@ -11,7 +11,7 @@ } from 'lucide-svelte' import { Button } from '$lib/components/common' import WorkspaceFamilyPicker from './WorkspaceFamilyPicker.svelte' - import { userStore, userWorkspaces, workspaceStore } from '$lib/stores' + import { isPremiumStore, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { goto } from '$lib/navigation' import { canCreateFork } from '$lib/utils/editInFork' import { isCloudHosted } from '$lib/cloud' @@ -52,9 +52,12 @@ const isFork = $derived(!!parentWorkspaceId) // Same gate as the sidebar WorkspaceMenu / SessionWorkspaceBar. - // When forking isn't available the diff/review surface is moot. + // When forking isn't available the diff/review surface is moot. On cloud, forking is a + // premium-only feature (backend caps it per paid seat). const forksAllowed = $derived( - !isCloudHosted() && canCreateFork($userStore) && $workspaceStore !== 'admins' + (!isCloudHosted() || $isPremiumStore) && + canCreateFork($userStore) && + $workspaceStore !== 'admins' ) let diffDrawer: ForkDiffDrawer | undefined = $state(undefined) diff --git a/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte b/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte index 6987730b87..91a5922871 100644 --- a/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceFamilyPicker.svelte @@ -2,6 +2,7 @@ import { tick, type Snippet } from 'svelte' import { enterpriseLicense, + isPremiumStore, userStore, userWorkspaces, workspaceStore, @@ -114,12 +115,14 @@ rootRulesetsResource.loading || rootUserInfoResource.loading || !canDeployRoot ) - // Structural gate: hidden on cloud, in the admins workspace, or when the user - // can't fork. DisableWorkspaceForking on the active workspace (a locked prod) - // doesn't apply when there's a dev to fork from instead — the dev isn't - // locked, and devOfRoot only resolves when the user is a member of it. + // Structural gate: hidden in the admins workspace, or when the user can't fork; on cloud, forking + // is premium-only (backend caps it per paid seat). DisableWorkspaceForking on the active workspace + // (a locked prod) doesn't apply when there's a dev to fork from instead — the dev isn't locked, and + // devOfRoot only resolves when the user is a member of it. const forksGateOpen = $derived( - !isCloudHosted() && $workspaceStore !== 'admins' && (canCreateFork($userStore) || !!devOfRoot) + (!isCloudHosted() || $isPremiumStore) && + $workspaceStore !== 'admins' && + (canCreateFork($userStore) || !!devOfRoot) ) // A fork is a new workspace, so it's subject to the community-edition cap on // the number of non-'admins' workspaces (backend _check_nb_of_workspaces, diff --git a/frontend/src/lib/components/settings/CloudQuotas.svelte b/frontend/src/lib/components/settings/CloudQuotas.svelte index 61158f13ec..ec597101ee 100644 --- a/frontend/src/lib/components/settings/CloudQuotas.svelte +++ b/frontend/src/lib/components/settings/CloudQuotas.svelte @@ -18,6 +18,7 @@ apps: QuotaInfo variables: QuotaInfo resources: QuotaInfo + forks: QuotaInfo } | undefined = $state(undefined) @@ -87,7 +88,8 @@ { label: 'Flows', key: 'flows', prunable: true }, { label: 'Apps', key: 'apps', prunable: true }, { label: 'Variables', key: 'variables', prunable: false }, - { label: 'Resources', key: 'resources', prunable: false } + { label: 'Resources', key: 'resources', prunable: false }, + { label: 'Forks', key: 'forks', prunable: false } ] diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index 71e5636512..53ac410cc8 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -217,7 +217,7 @@ {/if} - {#if !strictWorkspaceSelect && !isCloudHosted() && canCreateFork($userStore) && $workspaceStore !== 'admins'} + {#if !strictWorkspaceSelect && (!isCloudHosted() || $isPremiumStore) && canCreateFork($userStore) && $workspaceStore !== 'admins'}
{ - let gitSyncJobIds = await WorkspaceService.createWorkspaceForkGitBranch({ - workspace: $workspaceStore!, - requestBody: { - id: prefixed_id, - name, - color: colorEnabled && workspaceColor ? workspaceColor : undefined, - is_dev_workspace: createAsDevWorkspace, - // Send the lock intent in this first phase too so the backend can reject a non-admin's - // locked-dev request before any branch is created (avoids dangling branches). - lock_prod_deploy: createAsDevWorkspace && lockProdDeploy, - lock_prod_forking: createAsDevWorkspace && lockProdForking, - copy_members: copyMembers - } - }) + let gitSyncJobIds: string[] + try { + gitSyncJobIds = await WorkspaceService.createWorkspaceForkGitBranch({ + workspace: $workspaceStore!, + requestBody: { + id: prefixed_id, + name, + color: colorEnabled && workspaceColor ? workspaceColor : undefined, + is_dev_workspace: createAsDevWorkspace, + // Send the lock intent in this first phase too so the backend can reject a non-admin's + // locked-dev request before any branch is created (avoids dangling branches). + lock_prod_deploy: createAsDevWorkspace && lockProdDeploy, + lock_prod_forking: createAsDevWorkspace && lockProdForking, + copy_members: copyMembers + } + }) + } catch (e) { + // The backend can reject here (fork cap, depth limit, premium, non-admin lock). Reset the + // loading state and surface the error rather than leaving the button spinning. + forkCreationError = `Failed to create fork '${prefixed_id}'` + errorMsgs.push(e?.body ?? e ?? 'Unknown error') + forkCreationLoading = false + sendUserToast(`Could not create fork '${prefixed_id}' ${e?.body ?? e}`, true) + return + } try { await Promise.all( diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index e91ad87583..030cf67fd0 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -147,8 +147,15 @@ } catch (e) { console.error('Could not persist username to local storage', e) } - if (isCloudHosted() && user?.is_admin) { - isPremiumStore.set(await WorkspaceService.getIsPremium({ workspace })) + // Populate for all members (not just admins) so non-admin developers also get premium-gated + // affordances like the fork entry points on cloud. The `is_premium` endpoint is a boolean + // and no longer admin-gated. Best-effort: a failure here must not block user-store init. + if (isCloudHosted()) { + try { + isPremiumStore.set(await WorkspaceService.getIsPremium({ workspace })) + } catch (e) { + console.error('Could not fetch premium status', e) + } } } else { userStore.set(undefined) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 8fbec8f204..94628c8b17 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -1377,7 +1377,16 @@ /> {:else if tab == 'premium'} - + {#if currentWsForDevTab?.parent_workspace_id} + + This workspace is a fork of {currentWsForDevTab.parent_workspace_id}. It + runs on the parent's plan and its executions count toward the parent's usage and + bill, so there is no separate subscription here. Manage billing, seats, and quotas + from the parent workspace's settings. + + {:else} + + {/if} {:else if tab == 'slack'} Date: Thu, 2 Jul 2026 22:57:19 +0000 Subject: [PATCH 52/56] feat(s3): replace CE 50MB upload cap with 10GiB workspace storage quota (#9874) * fix(s3_proxy): enforce CE 50MB upload cap on multipart uploads Co-Authored-By: Claude Fable 5 * feat(s3): replace CE 50MB upload cap with 10GiB workspace storage quota Co-Authored-By: Claude Fable 5 * fix(s3): gate CE quota OSS stubs to not(enterprise) to match callers Co-Authored-By: Claude Fable 5 * fix(s3): delta-aware CE storage quota + guard usage-load retry loop Account for the overwritten object's size in the quota check so valid same-size overwrites near quota are not rejected (Codex review), and stop the storage-usage $effect from re-firing on persistent API errors (Pi review). Co-Authored-By: Claude Fable 5 * fix(s3): count chunked PUTs; revert overreaching volume quota copy Volumes write to workspace storage via a separate worker-side path with its own 50MB-per-file cap that this PR does not change, so revert the drawer copy that claimed they count toward the 10GiB quota (Codex review). Bump ee-repo-ref for the chunked-PUT accounting fix. Co-Authored-By: Claude Fable 5 * fix(s3): add SQLx cache for CE usage-bump/quota queries; exclude volumes Regenerate the missing offline SQLx cache for the not(enterprise) bump and remaining-quota queries so the private CE offline build compiles, and bump ee-repo-ref for the volumes/-prefix exclusion from the counted quota (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): always HEAD for CE upload delta so overwrites don't inflate usage Bump ee-repo-ref for the fast-path overwrite-accounting fix (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): reserve volumes/ prefix on CE write surfaces to close quota bypass Reject direct writes to the reserved volume prefix on the app-upload surface and add the OSS stub; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): refuse new multipart work when CE workspace is at quota Bump ee-repo-ref for the multipart-initiate/part quota gate (Codex review). Co-Authored-By: Claude Fable 5 * feat(s3): reserve in-flight multipart bytes against CE storage quota Add workspace_multipart_inflight table + grants, SQLx cache for the reservation queries, and bump ee-repo-ref. Bounds abandoned multipart uploads that the list-based recount can't see (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): clear multipart reservation only after a successful complete Add exclude-upload arg to the OSS quota stub/caller and the SQLx cache for the updated remaining-quota query; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * fix(s3): per-part multipart reservation; commit only on part success Per-part workspace_multipart_inflight schema (upload_id, part_id) so retries replace rather than double-count; SQLx cache for the reworked queries; bump ee-repo-ref (Codex review). Co-Authored-By: Claude Fable 5 * perf(s3): HEAD the multipart overwrite target once per upload, not per part SQLx cache for the stored-credit lookup; bump ee-repo-ref. Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to bea5a8b5120d6d69cab1ad4611ebe463559bd200 This commit updates the EE repository reference after PR #640 was merged in windmill-ee-private. Previous ee-repo-ref: 6e6ff86f1939cf74736b7d435bf6851416437523 New ee-repo-ref: bea5a8b5120d6d69cab1ad4611ebe463559bd200 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...464b5ff9f34e43fbcc38154939461aadfca9f.json | 18 +++ ...5a055b6972eb00e4dc1818f7a68bcf04c1ba7.json | 23 ++++ ...e0d5d1c3769d095352002daf8c11d10eac57b.json | 30 +++++ ...1bdb31f26e82fb0dc97bef7114d85959b6129.json | 35 ++++++ ...1cd50b297f80e5392715f54f32fab4bf346fc.json | 22 ++++ ...ebbe13e6298749b19570f9f8c4a0c7958b0ce.json | 36 ++++++ ...03ff9c96d613fb2d9b698361f4ede8031e5d2.json | 15 +++ ...01d9c16a9d4b5b422159fc239ff5502a7b71f.json | 15 +++ ...aaff40a634fc5530e202f65fad6a241f2671a.json | 19 +++ ...ed3e9625f54cd89cfd595665e4473c5f3ea6c.json | 37 ++++++ ...b610b028e88d5d2dbbea44377bf9dc416cd03.json | 22 ++++ ...5274749d7a436a753813da098758514a7493d.json | 22 ++++ ...6fea272ac289fc17ea9fc5aa237f77778b58f.json | 29 +++++ ...48af40192832c07c1c7049def3a80057bb683.json | 28 +++++ ...0ebf972bcc7f821fbf1e0171266efd8d46616.json | 16 +++ ...2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json | 34 +++++ ...406d1773657c62d67e55dac666f9469d50e6a.json | 15 +++ ...e7c80dd0ef92d78224792bec4c3323e14023c.json | 36 ++++++ ...05b693965f1bca6fef7418b19f98671c9204c.json | 16 +++ backend/ee-repo-ref.txt | 2 +- ...702064737_workspace_storage_usage.down.sql | 1 + ...60702064737_workspace_storage_usage.up.sql | 17 +++ ...3513_workspace_multipart_inflight.down.sql | 1 + ...213513_workspace_multipart_inflight.up.sql | 32 +++++ backend/windmill-api/openapi.yaml | 49 ++++++++ backend/windmill-api/src/apps.rs | 49 +++++++- backend/windmill-api/src/args.rs | 38 +++++- backend/windmill-api/src/job_helpers_oss.rs | 70 ++++++++++- .../workspaceSettings/StorageSettings.svelte | 119 ++++++++++++++++-- 29 files changed, 833 insertions(+), 13 deletions(-) create mode 100644 backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json create mode 100644 backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json create mode 100644 backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json create mode 100644 backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json create mode 100644 backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json create mode 100644 backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json create mode 100644 backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json create mode 100644 backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json create mode 100644 backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json create mode 100644 backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json create mode 100644 backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json create mode 100644 backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json create mode 100644 backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json create mode 100644 backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json create mode 100644 backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json create mode 100644 backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json create mode 100644 backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json create mode 100644 backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json create mode 100644 backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json create mode 100644 backend/migrations/20260702064737_workspace_storage_usage.down.sql create mode 100644 backend/migrations/20260702064737_workspace_storage_usage.up.sql create mode 100644 backend/migrations/20260702213513_workspace_multipart_inflight.down.sql create mode 100644 backend/migrations/20260702213513_workspace_multipart_inflight.up.sql diff --git a/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json new file mode 100644 index 0000000000..708922a0cb --- /dev/null +++ b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, storage, inflight_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, upload_id)\n DO UPDATE SET inflight_bytes = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f" +} diff --git a/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json new file mode 100644 index 0000000000..16063b9a3f --- /dev/null +++ b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7" +} diff --git a/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json new file mode 100644 index 0000000000..291dc00310 --- /dev/null +++ b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2 AND created_at > now() - ($3::text)::interval)::bigint as \"other_reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b" +} diff --git a/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json new file mode 100644 index 0000000000..137af7638b --- /dev/null +++ b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129" +} diff --git a/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json new file mode 100644 index 0000000000..18af751ca0 --- /dev/null +++ b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('workspace_multipart_inflight'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc" +} diff --git a/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json new file mode 100644 index 0000000000..cd2dd23319 --- /dev/null +++ b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3))::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce" +} diff --git a/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json new file mode 100644 index 0000000000..be54b4a25d --- /dev/null +++ b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2" +} diff --git a/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json new file mode 100644 index 0000000000..7250aff598 --- /dev/null +++ b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at < now() - ($2::text)::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f" +} diff --git a/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json new file mode 100644 index 0000000000..c0207b6bc7 --- /dev/null +++ b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, part_id, storage, part_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, upload_id, part_id)\n DO UPDATE SET part_bytes = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a" +} diff --git a/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json new file mode 100644 index 0000000000..253fe79753 --- /dev/null +++ b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n -- reservation of every OTHER in-flight upload\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2\n AND created_at > now() - ($4::text)::interval\n GROUP BY upload_id) t)::bigint as \"other_reserved!\",\n -- this upload's already-recorded parts, excluding the part being (re)uploaded\n (SELECT COALESCE(SUM(part_bytes), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 AND part_id <> $3)::bigint as \"this_other_parts!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "this_other_parts!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c" +} diff --git a/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json new file mode 100644 index 0000000000..0c3fdb8d0a --- /dev/null +++ b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_xact_lock(hashtext('workspace_storage_usage'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_xact_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03" +} diff --git a/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json new file mode 100644 index 0000000000..691016a610 --- /dev/null +++ b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') as \"key!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d" +} diff --git a/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json new file mode 100644 index 0000000000..0f1bcc2271 --- /dev/null +++ b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT inflight_bytes, target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inflight_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f" +} diff --git a/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json new file mode 100644 index 0000000000..1148679230 --- /dev/null +++ b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(bytes), 0)::bigint as \"total!\",\n COALESCE(MIN(computed_at) < now() - interval '10 minutes', true) as \"stale!\"\n FROM workspace_storage_usage WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683" +} diff --git a/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json new file mode 100644 index 0000000000..96608e1bc2 --- /dev/null +++ b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, GREATEST($3::bigint, 0), to_timestamp(0))\n ON CONFLICT (workspace_id, storage)\n DO UPDATE SET bytes = GREATEST(workspace_storage_usage.bytes + $3::bigint, 0)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616" +} diff --git a/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json new file mode 100644 index 0000000000..abb128f463 --- /dev/null +++ b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT storage, bytes, computed_at FROM workspace_storage_usage\n WHERE workspace_id = $1 ORDER BY storage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "storage", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "computed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b" +} diff --git a/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json new file mode 100644 index 0000000000..1fd1fbaece --- /dev/null +++ b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_storage_usage WHERE workspace_id = $1 AND storage != ALL($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a" +} diff --git a/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json new file mode 100644 index 0000000000..5609adaad8 --- /dev/null +++ b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3)\n GROUP BY upload_id) t)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c" +} diff --git a/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json new file mode 100644 index 0000000000..4f0731f5e4 --- /dev/null +++ b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, $3, now())\n ON CONFLICT (workspace_id, storage) DO UPDATE SET bytes = $3, computed_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5e76f395d2..45adcf5f9f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c3852ecb36bd0be1a74c63169e513888f3347850 +bea5a8b5120d6d69cab1ad4611ebe463559bd200 diff --git a/backend/migrations/20260702064737_workspace_storage_usage.down.sql b/backend/migrations/20260702064737_workspace_storage_usage.down.sql new file mode 100644 index 0000000000..e32480a602 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_storage_usage; diff --git a/backend/migrations/20260702064737_workspace_storage_usage.up.sql b/backend/migrations/20260702064737_workspace_storage_usage.up.sql new file mode 100644 index 0000000000..d7274e8742 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.up.sql @@ -0,0 +1,17 @@ +-- Cached per-(workspace, storage) byte usage of workspace object storage, +-- refreshed by listing the storage location and adjusted optimistically as +-- uploads complete. Read on every workspace-storage write in CE builds to +-- enforce the storage quota, and by the storage_usage endpoint in all builds. +CREATE TABLE workspace_storage_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + storage VARCHAR(255) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, storage) +); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role +-- that set them (same gap as workspace_diff, notify_event, script_trigger). +GRANT ALL ON workspace_storage_usage TO windmill_user; +GRANT ALL ON workspace_storage_usage TO windmill_admin; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql new file mode 100644 index 0000000000..e72edd0b12 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_multipart_inflight; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql new file mode 100644 index 0000000000..9768eaf842 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql @@ -0,0 +1,32 @@ +-- Reservation for the parts of in-flight (initiated but not yet completed) +-- multipart uploads to workspace object storage. Uncommitted parts occupy +-- object-store capacity but are invisible to the list-based storage recount +-- until completion, so CE folds this reservation into the remaining quota to +-- bound abandoned uploads. One row per uploaded part so a re-uploaded part +-- (same part_id) replaces rather than double-counts; a part is recorded only +-- after its upstream upload succeeds. Rows are removed on successful complete +-- and lazily expired after a TTL (abort/abandon rely on the TTL, which matches +-- when the object store reaps the uncommitted parts). +-- part_id - S3 part number or Azure block id (string) +-- part_bytes - size of that part +-- target_existing_size - size of the object the upload will overwrite (0 if new), +-- credited so an overwrite only reserves the net growth +CREATE TABLE workspace_multipart_inflight ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + upload_id VARCHAR(512) NOT NULL, + part_id VARCHAR(256) NOT NULL, + storage VARCHAR(255) NOT NULL, + part_bytes BIGINT NOT NULL DEFAULT 0, + target_existing_size BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, upload_id, part_id) +); + +CREATE INDEX idx_workspace_multipart_inflight_created_at + ON workspace_multipart_inflight (created_at); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role that +-- set them (same gap as workspace_storage_usage, notify_event, script_trigger). +GRANT ALL ON workspace_multipart_inflight TO windmill_user; +GRANT ALL ON workspace_multipart_inflight TO windmill_admin; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index eec76ea009..e661c3aeb8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -19901,6 +19901,55 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/storage_usage: + get: + summary: Get the storage usage of the workspace object storage, per configured storage. On Community Edition, also returns the workspace storage quota. + operationId: getStorageUsage + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: refresh + in: query + description: recount usage by listing the storage instead of returning cached values + schema: + type: boolean + responses: + "200": + description: Storage usage + content: + application/json: + schema: + type: object + properties: + total_bytes: + type: integer + format: int64 + quota_bytes: + type: integer + format: int64 + description: only present on Community Edition, where workspace storage is capped + storages: + type: array + items: + type: object + properties: + storage: + type: string + bytes: + type: integer + format: int64 + computed_at: + type: string + format: date-time + required: + - storage + - bytes + - computed_at + required: + - total_bytes + - storages + /w/{workspace}/job_helpers/list_stored_files: get: summary: List the file keys available in a workspace object storage diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 38fa5ff862..99d6156687 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,6 +7,11 @@ use std::{collections::HashMap, sync::Arc}; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ +#[cfg(all(feature = "parquet", not(feature = "enterprise")))] +use crate::job_helpers_oss::{ + bump_storage_usage, ce_upload_budget, reject_reserved_volume_key, + spawn_storage_usage_recount_floored, +}; use crate::{ auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, @@ -3544,7 +3549,49 @@ async fn upload_s3_file_from_app( ]) .into(); - let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?; + // Only workspace storage is quota-metered; a custom-resource upload lands in + // the user's own bucket and is neither capped nor counted. An overwrite of an + // existing key only spends the difference over its current size. + let _is_workspace_storage = query.s3_resource_path.is_none(); + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + reject_reserved_volume_key(&file_key)?; + } + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + let (max_size, _existing_size) = if _is_workspace_storage { + let content_length = request + .headers() + .get(http::header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()); + let budget = ce_upload_budget(&db, &w_id, &s3_client, &file_key, content_length).await?; + (Some(budget.max_size), budget.existing_size) + } else { + (None, 0) + }; + #[cfg(any(not(feature = "parquet"), feature = "enterprise"))] + let max_size: Option = None; + + match upload_file_from_req(s3_client, &file_key, request, options, max_size).await { + Ok((_, _size)) => + { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64 - _existing_size, + ) + .await; + } + } + Err(e) => { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + spawn_storage_usage_recount_floored(&db, &w_id); + return Err(e); + } + } let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims { file_key: file_key.clone(), diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 3793e94b02..dc8b36fbaf 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -90,6 +90,10 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { + #[cfg(not(feature = "enterprise"))] + use crate::job_helpers_oss::{ + bump_storage_usage, ce_storage_quota_remaining, spawn_storage_usage_recount_floored, + }; use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; @@ -139,8 +143,38 @@ impl RawWebhookArgs { .into_stream() .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); - upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) - .await?; + // file_key is always freshly random here, so this never + // overwrites an existing object; the full size is the delta. + #[cfg(not(feature = "enterprise"))] + let max_size = Some(ce_storage_quota_remaining(db, w_id, None).await? as usize); + #[cfg(feature = "enterprise")] + let max_size: Option = None; + + match upload_file_internal( + s3_client.clone(), + &file_key, + bytes_stream, + options, + max_size, + ) + .await + { + Ok((_, _size)) => { + #[cfg(not(feature = "enterprise"))] + bump_storage_usage( + db, + w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64, + ) + .await; + } + Err(e) => { + #[cfg(not(feature = "enterprise"))] + spawn_storage_usage_recount_floored(db, w_id); + return Err(e); + } + } files.entry(name).or_insert(vec![]).push(serde_json::json!({ "s3": &file_key diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 23f20459c3..c6d8e397f2 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -87,7 +87,8 @@ pub async fn upload_file_from_req( _file_key: &str, _req: axum::extract::Request, _options: PutMultipartOpts, -) -> error::Result { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) @@ -99,12 +100,77 @@ pub async fn upload_file_internal( _file_key: &str, _stream: impl Stream> + Unpin, _options: PutMultipartOpts, -) -> error::Result<()> { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } +// These stubs stand in for the CE quota helpers in a pure-OSS build; their only +// callers (apps.rs / args.rs uploads) are `not(enterprise)`, so gate them the +// same way — an enterprise-without-private build compiles neither. +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_storage_quota_remaining( + _db: &DB, + _w_id: &str, + _exclude_upload_id: Option<&str>, +) -> error::Result { + Ok(i64::MAX) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn reject_reserved_volume_key(_file_key: &str) -> error::Result<()> { + Ok(()) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub struct CeUploadBudget { + pub max_size: usize, + pub existing_size: i64, +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_upload_budget( + _db: &DB, + _w_id: &str, + _s3_client: &Arc, + _file_key: &str, + _content_length: Option, +) -> error::Result { + Ok(CeUploadBudget { max_size: usize::MAX, existing_size: 0 }) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn bump_storage_usage(_db: &DB, _w_id: &str, _storage: &str, _delta: i64) {} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn spawn_storage_usage_recount_floored(_db: &DB, _w_id: &str) {} + #[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: OptJobAuthed, diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index 329ea53e14..be49e3c564 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -1,7 +1,7 @@ + + (open = false)}> + (open = false)}> + {#snippet actions()} +
+ {#each macros as m (m.name)} +
+
+ + {macroSignature(m)} + + + {m.is_table ? 'table' : 'scalar'} + +
+ 
{m.body}
+
+ {/each} + + {/each} + + {/if} + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index c709866a5f..8b5e90170c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -10,6 +10,7 @@ Loader2, Play, RotateCw, + SquareFunction, Tag, Target, Timer, @@ -37,6 +38,9 @@ freshness?: string tag?: string retry?: { count: number; delay?: string } + // Macros this script provides (deployed/drafted `// macros` library). + // Non-empty renders the ƒ chip marking the node as a macro library. + macros?: { name: string; params: string; is_table: boolean }[] // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState @@ -199,6 +203,17 @@ ×{r.count} {/if} + {#if data.macros && data.macros.length > 0} +
1 ? 's' : ''}:\n${data.macros + .map((m) => `• ${m.name}(${m.params})${m.is_table ? ' → table' : ''}`) + .join('\n')}`} + > + + ×{data.macros.length} +
+ {/if} {#if data.runState} {@const rs = data.runState}
= { retry: true, materialize: true, dataTests: true, - columnLineage: true + columnLineage: true, + macros: true, + useLibs: true } // Parser-parity guard: this TS parser (drives the live graph preview) and @@ -78,6 +80,10 @@ type Fixture = { // Snake_case `ColumnLineage` serde shape — TS parser emits it verbatim, // so the comparison is 1:1. Absent === []. column_lineage?: Array> + // `// macros` marker. Absent === false. + macros?: boolean + // `// use ` accumulation, declaration order, deduped. Absent === []. + use_libs?: string[] } } @@ -179,6 +185,10 @@ describe('parsePipelineAnnotations matches the shared Rust fixture corpus', () = expect(got.dataTests, 'data tests').toEqual(f.expected.data_tests ?? []) expect(got.columnLineage, 'column lineage').toEqual(f.expected.column_lineage ?? []) + + expect(got.macros, 'macros').toBe(f.expected.macros ?? false) + + expect(got.useLibs, 'use_libs').toEqual(f.expected.use_libs ?? []) }) } }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 5f4b063e3f..315c7526f3 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -95,6 +95,33 @@ describe('parsePipelineAnnotations: retry', () => { }) }) +describe('parsePipelineAnnotations: macros + use', () => { + it('parses the bare macros marker', () => { + const out = parsePipelineAnnotations('// macros\nCREATE MACRO m(a) AS a;') + expect(out.macros).toBe(true) + }) + + it('macros marker is strict — trailing prose and variants rejected', () => { + expect(parsePipelineAnnotations('// macros are defined below\n').macros).toBe(false) + expect(parsePipelineAnnotations('// macros_v2\n').macros).toBe(false) + expect(parsePipelineAnnotations('-- macros \nSELECT 1;').macros).toBe(true) + }) + + it('use accumulates in order and dedups', () => { + const out = parsePipelineAnnotations( + '// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\n' + ) + expect(out.useLibs).toEqual(['f/lib/stats', 'f/lib/dates']) + }) + + it('use rejects prose, slashless and multi-token values', () => { + const out = parsePipelineAnnotations( + '// use this script to compute\n// use standalone\n// use f/lib/ok extra\n' + ) + expect(out.useLibs).toEqual([]) + }) +}) + describe('parsePipelineAnnotations: combined', () => { it('parses all keywords together', () => { const code = [ diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index f58ffa54ad..f33a03fff5 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -173,6 +173,12 @@ export type PipelineAnnotations = { dataTests: DataTest[] // `// column <- .
[, …]` — accumulating column lineage. columnLineage: ColumnLineage[] + // Bare `// macros` (alone on the line, like `// pipeline`) — marks this + // DuckDB script as a workspace macro library. + macros: boolean + // `// use ` — force-inject the named macro library into + // this script's jobs. Accumulating, declaration order, deduped. + useLibs: string[] } // Tokenize a `key=value [key="quoted value"] ...` option string. Bare @@ -510,7 +516,9 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { triggerAssets: [], nativeTriggers: [], dataTests: [], - columnLineage: [] + columnLineage: [], + macros: false, + useLibs: [] } for (const rawLine of code.split('\n')) { @@ -530,6 +538,26 @@ export function parsePipelineAnnotations(code: string): PipelineAnnotations { continue } + const afterMacros = consumeKeyword(inner, 'macros') + if (afterMacros !== undefined) { + // Strict like `pipeline`: keyword alone on the line, so prose such + // as `// macros are defined below` never false-positives. + if (afterMacros.trim() === '') out.macros = true + continue + } + + // `// use ` — accumulating. The argument must be a + // single whitespace-free token containing `/` (all script paths do), + // so prose like `// use this script to …` is dropped fail-safe. + const afterUse = consumeKeyword(inner, 'use') + if (afterUse !== undefined) { + const path = afterUse.trim() + if (path && !/\s/.test(path) && path.includes('/') && !out.useLibs.includes(path)) { + out.useLibs.push(path) + } + continue + } + const afterPart = consumeKeyword(inner, 'partitioned') if (afterPart !== undefined) { if (!out.partition) { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index cbb5476516..71a09318e6 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -19,6 +19,7 @@ export type PipelineOutputKind = | 'materialize' | 's3_parquet' | 's3_object' + | 'macros' export type PipelineOutputKindMeta = { id: PipelineOutputKind @@ -56,6 +57,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'S3 Object', description: 'Generic file (JSON/CSV/binary)' }, + { + id: 'macros', + label: 'Macro library', + description: 'Reusable DuckDB macros, callable from every script in the workspace' + }, { id: 'none', label: 'No output', @@ -77,7 +83,7 @@ const LANG_COMPATIBILITY: Record = { // single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL // SELECT (not in-memory rows), so a polyglot managed materialize is a // separate follow-up — those langs keep the `ducklake` raw-write kind. - duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'none'], + duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'macros', 'none'], postgresql: ['datatable', 'none'], mysql: ['none'], mssql: ['none'], @@ -176,6 +182,9 @@ export function autoOutputAsset( path: `/pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` } } + // A macro library produces no asset — its "output" is the registry + // entries the deploy records. + case 'macros': case 'none': return undefined } @@ -330,6 +339,16 @@ function header(ctx: TemplateContext): string { `${p} Strategy: add key=to merge (upsert), or append for insert-only; default replaces the partition` ] : [] + // Macro library: the `// macros` marker registers every CREATE MACRO below + // into the workspace registry at deploy. The hint must not start with a + // parser keyword — `Consumers` is safe. + const macrosLine = + outputKind === 'macros' + ? [ + `${p} macros`, + `${p} Consumers just call these by name; add \`${p} use \` in a consumer to force-inject the whole library` + ] + : [] // Discoverability hint — the three annotations users most often miss // when authoring their first pipeline script. Single line, real // example values (not placeholders) so users see the syntax. Docs @@ -337,7 +356,7 @@ function header(ctx: TemplateContext): string { // line separates it from the parsed annotations above (`// pipeline`, // `// on …`) so the editor reads as "real annotations, then a hint". const more = `${p} More: partitioned daily, freshness 1h, retry 3, tag heavy — https://www.windmill.dev/docs/pipelines/annotations` - return [`${p} pipeline`, ...lines, ...matLine, '', more, ''].join('\n') + return [`${p} pipeline`, ...lines, ...matLine, ...macrosLine, '', more, ''].join('\n') } // Bun / Deno bodies. These share the wmill SDK surface, so we treat them @@ -636,6 +655,18 @@ function bodyDuckdb(ctx: TemplateContext): string { ) } break + case 'macros': + // Library body: only CREATE [OR REPLACE] MACRO statements (plus plain + // setup). One scalar + one table example; bodies may only call macros + // defined EARLIER in the file (DuckDB bind-checks at creation). + lines.push( + `CREATE OR REPLACE MACRO safe_div(a, b, fallback := 0) AS`, + ` CASE WHEN b = 0 THEN fallback ELSE a / b END;`, + ``, + `CREATE OR REPLACE MACRO sample_rows(src, n) AS TABLE`, + ` SELECT * FROM query_table(src) LIMIT n;` + ) + break case 'none': default: // With an uploaded file but no output asset, at least surface its diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts index 378c4e1469..3c98d57602 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts @@ -9,6 +9,9 @@ const ann = (over: Partial = {}): PipelineAnnotations => ({ triggerAssets: [], nativeTriggers: [], dataTests: [], + columnLineage: [], + macros: false, + useLibs: [], ...over }) @@ -452,4 +455,85 @@ describe('resolveGraph', () => { missing: true }) }) + + it('macro edges: base passes through; live `// use` adds an unsaved via_use edge', () => { + const base = baseGraph({ + runnables: [ + { + path: 'f/lib/stats', + usage_kind: 'script', + macros: [{ name: 'safe_div', params: 'a, b', is_table: false }] + }, + { path: 'f/x/cons', usage_kind: 'script' } + ], + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { + scriptPath: 'f/x/other', + annotations: ann({ useLibs: ['f/lib/stats'] }) + } + }) + ) + // Detection edge preserved untouched. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: false + }) + // Live `// use` synthesizes an unsaved whole-lib edge with the lib's names. + expect(r.macro_edges).toContainEqual({ + lib_path: 'f/lib/stats', + consumer_path: 'f/x/other', + macro_names: ['safe_div'], + via_use: true, + unsaved: true + }) + }) + + it('macro edges: removing the `// use` line of an overlaid consumer retires its via_use edge', () => { + const base = baseGraph({ + macro_edges: [ + { + lib_path: 'f/lib/stats', + consumer_path: 'f/x/cons', + macro_names: ['safe_div'], + via_use: true + } + ] + }) + const r = resolveGraph( + input({ + base, + liveAnnotations: { scriptPath: 'f/x/cons', annotations: ann() } + }) + ) + expect(r.macro_edges).toEqual([]) + }) + + it('macro edges: draft `// macros` library gets the ƒ badge data from its body', () => { + const drafts = new Map([ + [ + 'f/lib/new', + { + script: { + content: '// macros\nCREATE OR REPLACE MACRO dbl(a) AS a * 2;' + } + } + ] + ]) + const r = resolveGraph(input({ drafts })) + const lib = r.runnables.find((x) => x.path === 'f/lib/new') + expect(lib?.macros).toEqual([{ name: 'dbl', params: 'a', is_table: false }]) + }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts index e223379adb..849849c555 100644 --- a/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts +++ b/frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts @@ -1,4 +1,4 @@ -import type { AssetGraphResponse, NativeTriggerKind } from './types' +import type { AssetGraphMacroEdge, AssetGraphResponse, NativeTriggerKind } from './types' import { mergeColumnLineage, parsePipelineAnnotations, @@ -146,15 +146,94 @@ export function resolveGraph(input: ResolveGraphInput): AssetGraphResponse { return false return true }) + // Mirror the backend's skip-if-empty: no `macro_edges` key at all when + // there is nothing to show (also keeps the no-macros response shape + // byte-identical to before the feature). + const macroEdges = resolveMacroEdges(input) return { ...base, assets: acc.assets, runnables: acc.runnables, edges: acc.edges, - triggers: [...baseTriggers, ...acc.extraTriggers] + triggers: [...baseTriggers, ...acc.extraTriggers], + ...(macroEdges.length > 0 || base.macro_edges ? { macro_edges: macroEdges } : {}) } } +/** + * Macro-library → consumer edges: the deployed base edges, with `// use` + * declarations of overlaid scripts (drafts + the open buffer) taking over + * their consumer's `via_use` edges so adding/removing a `// use` line updates + * the canvas live. Detection-based edges (macro calls in the deployed body) + * are backend-owned and only refresh on redeploy. + */ +function resolveMacroEdges(input: ResolveGraphInput): AssetGraphMacroEdge[] { + const { base, drafts, liveAnnotations } = input + const libMacroNames = new Map() + for (const r of base.runnables) { + if (r.usage_kind === 'script' && r.macros?.length) { + libMacroNames.set( + r.path, + r.macros.map((m) => m.name) + ) + } + } + const useByPath = new Map() + for (const [path, d] of drafts) { + useByPath.set(path, parsePipelineAnnotations(d.script.content).useLibs) + } + if (liveAnnotations.scriptPath) { + // `?? []` — callers may hand a minimal annotations object (tests, older + // call sites) that predates the field. + useByPath.set(liveAnnotations.scriptPath, liveAnnotations.annotations.useLibs ?? []) + } + const out: AssetGraphMacroEdge[] = [] + for (const e of base.macro_edges ?? []) { + if (e.via_use && useByPath.has(e.consumer_path)) continue + out.push({ ...e }) + } + for (const [path, libs] of useByPath) { + for (const lib of libs) { + const existing = out.find((e) => e.lib_path === lib && e.consumer_path === path) + if (existing) { + // Upgrade the detection edge in place: `// use` pulls in the whole + // library, so the edge covers every macro the lib defines. + existing.via_use = true + existing.unsaved = true + existing.macro_names = [ + ...new Set([...existing.macro_names, ...(libMacroNames.get(lib) ?? [])]) + ] + } else { + out.push({ + lib_path: lib, + consumer_path: path, + macro_names: libMacroNames.get(lib) ?? [], + via_use: true, + unsaved: true + }) + } + } + } + return out +} + +// Light regex extraction of a draft macro library's definitions for the live +// node badge. The strict grammar lives in the Rust `parse_macro_library` at +// deploy; this only needs name/params/table-ness for display (nested parens +// in a default value may truncate the shown signature, never the deploy). +const MACRO_DEF_RE = + /create\s+(?:or\s+replace\s+)?(?:temp(?:orary)?\s+)?(?:macro|function)\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*\(([^)]*)\)\s*as\s+(table\b)?/gi + +export function extractDraftMacros( + content: string +): { name: string; params: string; is_table: boolean }[] { + const out: { name: string; params: string; is_table: boolean }[] = [] + for (const m of content.matchAll(MACRO_DEF_RE)) { + out.push({ name: m[1].toLowerCase(), params: m[2].trim(), is_table: m[3] !== undefined }) + } + return out +} + type ResolveContext = { draftedPaths: Set isDrafted: (kind: string, p: string) => boolean @@ -251,6 +330,9 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { // duplicate (which would crash svelte-flow's keyed each), so the // canvas + trigger-node labels reflect that there's pending body // editing for this path. + // `// macros` library draft: extract the definitions for the live node + // badge (regex-light; the strict parse happens at deploy). + const draftMacros = parsed.macros ? extractDraftMacros(d.script.content) : [] const baseIdx = runnables.findIndex((r) => r.usage_kind === 'script' && r.path === path) if (baseIdx === -1) { runnables.push({ @@ -264,6 +346,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true }) } else { @@ -276,6 +359,7 @@ function seedDraftOverlays(acc: Accumulator, input: ResolveGraphInput) { data_tests: parsed.dataTests.length > 0 ? parsed.dataTests : undefined, column_lineage: mergedCL.length > 0 ? mergedCL : undefined, materialize_target: materializeTarget, + macros: draftMacros.length > 0 ? draftMacros : undefined, unsaved: true } } diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 274d218e82..0db259a389 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -46,6 +46,11 @@ export interface AssetGraphRunnableNode { // produced asset's schema can evolve: only whole-table `replace` can, since // `append`/`merge`/partitioned writes INSERT into a fixed-schema table. materialize_strategy?: 'replace' | 'append' | 'merge' + // Macros this script provides to the workspace registry (deployed + // `// macros` library). Non-empty marks the node as a macro library; + // drives the "defines N macros" badge and the details-pane signature + // list. `params` is the verbatim parameter list. + macros?: { name: string; params: string; is_table: boolean }[] // Synthesized by the page from a local draft; the script doesn't exist // in the DB yet. Drives a dashed/lower-opacity rendering to mirror how // unsaved triggers are styled — visually distinct from persisted nodes. @@ -113,11 +118,24 @@ export type AssetGraphTrigger = missing?: boolean } +// Macro-library → consumer edge: the consumer calls `macro_names` of +// `lib_path`'s macros (deploy-recorded detection), or pulls in the whole +// library via `// use` (`via_use`, macro_names then lists the full library). +// `unsaved: true` marks a draft's `// use` overlay. +export interface AssetGraphMacroEdge { + lib_path: string + consumer_path: string + macro_names: string[] + via_use: boolean + unsaved?: boolean +} + export interface AssetGraphResponse { assets: AssetGraphAssetNode[] runnables: AssetGraphRunnableNode[] edges: AssetGraphEdge[] triggers: AssetGraphTrigger[] + macro_edges?: AssetGraphMacroEdge[] } export type AssetGraphNodeData = diff --git a/frontend/src/lib/components/assets/workspaceMacros.ts b/frontend/src/lib/components/assets/workspaceMacros.ts new file mode 100644 index 0000000000..d25f9d1bea --- /dev/null +++ b/frontend/src/lib/components/assets/workspaceMacros.ts @@ -0,0 +1,32 @@ +import { AssetService, type ListWorkspaceMacrosResponse } from '$lib/gen' + +export type WorkspaceMacro = ListWorkspaceMacrosResponse[number] + +// Workspace macros are late-bound (the worker reads the registry per job), so +// mild staleness in editor surfaces is harmless — a short TTL keeps repeated +// editor mounts / drawer opens from refetching on every keystroke-driven +// remount while still picking up a lib deploy within seconds. +const TTL_MS = 30_000 +const cache = new Map() + +export async function listWorkspaceMacrosCached(workspace: string): Promise { + const hit = cache.get(workspace) + if (hit && Date.now() - hit.at < TTL_MS) return hit.items + const items = await AssetService.listWorkspaceMacros({ workspace }) + cache.set(workspace, { at: Date.now(), items }) + return items +} + +export function invalidateWorkspaceMacros(workspace: string) { + cache.delete(workspace) +} + +/** `name(params)` display signature, with the table-macro arrow. */ +export function macroSignature(m: WorkspaceMacro): string { + return `${m.name}(${m.params})${m.is_table ? ' → table' : ''}` +} + +/** Full `CREATE` statement for the copy button / documentation preview. */ +export function macroDefinitionSql(m: WorkspaceMacro): string { + return `CREATE OR REPLACE MACRO ${m.name}(${m.params}) AS ${m.is_table ? 'TABLE ' : ''}${m.body};` +} diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 47b4bf6abc..4d41f7ff05 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -29,6 +29,7 @@ PipelineMode } from '$lib/components/assets/AssetGraph/types' import PipelineModeToggle from '$lib/components/assets/AssetGraph/PipelineModeToggle.svelte' + import MacroExplorerDrawer from '$lib/components/assets/AssetGraph/MacroExplorerDrawer.svelte' import { parsePipelineAnnotations } from '$lib/components/assets/AssetGraph/parsePipelineAnnotations' import { buildColumnGraph, @@ -83,6 +84,7 @@ Play, RefreshCw, Save, + SquareFunction, Target, Telescope } from 'lucide-svelte' @@ -218,6 +220,24 @@ } } + // Workspace-macro explorer drawer (all `// macros` libraries + their + // signatures/bodies). "Open" on a group selects the lib node when it's on + // this canvas; a lib living in another folder opens its script page. + let macroDrawer: MacroExplorerDrawer | undefined = $state() + function openMacroLib(path: string) { + const onCanvas = displayGraph?.runnables?.some( + (r) => r.usage_kind === 'script' && r.path === path + ) + if (onCanvas) { + pe.selection = { kind: 'runnable', runnable_kind: 'script', path } + pe.activeDraftPath = undefined + panelHidden = false + focusPipelineNode(`script:${path}`) + } else { + window.open(`${base}/scripts/get/${path}`, '_blank') + } + } + // Draft autosave (the data_pipeline DraftService bundle) lives inside // PipelineGraphEditor now; the route just supplies its path to the indicator. let pipelineDraftPath = $derived(`f/${folder}/data_pipeline`) @@ -233,7 +253,9 @@ triggerAssets: [], nativeTriggers: [], dataTests: [], - columnLineage: [] + columnLineage: [], + macros: false, + useLibs: [] } } @@ -2105,6 +2127,15 @@ Activity {/if} +