diff --git a/.gitignore b/.gitignore index 5f733611de..e80b4f32ca 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ backend/chrome_profiler.json .fast-check/ __pycache__/ .playwright-mcp/ -.codex \ No newline at end of file +.codex +.claude/scheduled_tasks.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index 8441043de0..73f6ba3f12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,139 @@ # Changelog +## [1.734.0](https://github.com/windmill-labs/windmill/compare/v1.733.1...v1.734.0) (2026-06-20) + + +### Features + +* ducklake materialization for data pipelines ([#9689](https://github.com/windmill-labs/windmill/issues/9689)) ([3ebf243](https://github.com/windmill-labs/windmill/commit/3ebf24359d66048d6361ce65cd879cdc04b737ed)) + + +### Bug Fixes + +* **frontend:** clear branch step state when switching outer loop iterations ([#9650](https://github.com/windmill-labs/windmill/issues/9650)) ([09a8004](https://github.com/windmill-labs/windmill/commit/09a80040ca268a4379d5302e3401435ba93247e0)) + +## [1.733.1](https://github.com/windmill-labs/windmill/compare/v1.733.0...v1.733.1) (2026-06-19) + + +### Bug Fixes + +* **backend:** validate ansible vault_id entries before config generation ([#9681](https://github.com/windmill-labs/windmill/issues/9681)) ([c1f31c0](https://github.com/windmill-labs/windmill/commit/c1f31c0e4777bf0cfed0dd7f03249e9a61cd8cb9)) +* **frontend:** group live pipeline runs in the activity panel ([#9684](https://github.com/windmill-labs/windmill/issues/9684)) ([1be4df9](https://github.com/windmill-labs/windmill/commit/1be4df9acb935250d4cc12e83cf67e366d870d5a)) +* require super admin for object storage config test endpoint ([#9683](https://github.com/windmill-labs/windmill/issues/9683)) ([fb44fe7](https://github.com/windmill-labs/windmill/commit/fb44fe7af2b8ebe8ef64ffb0e5acce8580bf4200)) +* validate websocket trigger urls and gate trigger test route ([#9682](https://github.com/windmill-labs/windmill/issues/9682)) ([c39ee07](https://github.com/windmill-labs/windmill/commit/c39ee07c0bcd2249dd19ffa5cd988125eefc6c9f)) + +## [1.733.0](https://github.com/windmill-labs/windmill/compare/v1.732.0...v1.733.0) (2026-06-19) + + +### Features + +* **ai-chat:** cap read_app_file + search_app grep tool to bound context in large raw apps ([#9653](https://github.com/windmill-labs/windmill/issues/9653)) ([4296a6a](https://github.com/windmill-labs/windmill/commit/4296a6ae1f73564de4df54fe1df0a03c1df05dfd)) +* **python, windows:** enable S3 to cache wheels ([#5199](https://github.com/windmill-labs/windmill/issues/5199)) ([ab3bc97](https://github.com/windmill-labs/windmill/commit/ab3bc97cd92b6480327029bcf018280442462af7)) + + +### Bug Fixes + +* allow users to always discard their own drafts without write permission ([#9659](https://github.com/windmill-labs/windmill/issues/9659)) ([6833a55](https://github.com/windmill-labs/windmill/commit/6833a554aeddb3e63173d3c3140b490c0bf2822b)) +* **backend:** clean up unique_ext_jwt_token on workspace deletion ([#9676](https://github.com/windmill-labs/windmill/issues/9676)) ([9add719](https://github.com/windmill-labs/windmill/commit/9add719d936cdcfb2c4062629e3e1f792694dafe)) +* **backend:** strip NUL bytes from draft values on write ([#9673](https://github.com/windmill-labs/windmill/issues/9673)) ([924f9c7](https://github.com/windmill-labs/windmill/commit/924f9c7e8d8863d9af40aee246a519b4be0e1ea2)) +* **python:** split PIP_TRUSTED_HOST by whitespace to support multiple hosts ([#9675](https://github.com/windmill-labs/windmill/issues/9675)) ([cafb473](https://github.com/windmill-labs/windmill/commit/cafb473494d9cff3a8b2aeaf9f18b015f966e7b3)) + +## [1.732.0](https://github.com/windmill-labs/windmill/compare/v1.731.0...v1.732.0) (2026-06-19) + + +### Features + +* **ansible:** add AI chat and editor bar buttons for ansible ([#9671](https://github.com/windmill-labs/windmill/issues/9671)) ([017c3d3](https://github.com/windmill-labs/windmill/commit/017c3d3343c2577501103be4b2dd8dac9727d80d)) + + +### Bug Fixes + +* **ai:** emit token usage in gemini proxy streaming translation ([#9669](https://github.com/windmill-labs/windmill/issues/9669)) ([0cc2257](https://github.com/windmill-labs/windmill/commit/0cc2257596a3965cf6db21a0090edcce6e1b8419)) +* **backend:** grant script_trigger access to windmill roles ([#9674](https://github.com/windmill-labs/windmill/issues/9674)) ([3361736](https://github.com/windmill-labs/windmill/commit/33617367d09537667d2ab3f91135c736194b9e7e)) +* **frontend:** ignore hash/assets in script diffs and drafts (WIN-2071) ([#9664](https://github.com/windmill-labs/windmill/issues/9664)) ([3371265](https://github.com/windmill-labs/windmill/commit/33712653821e83f2562dd5f271dbec0188d5d2f8)) + +## [1.731.0](https://github.com/windmill-labs/windmill/compare/v1.730.0...v1.731.0) (2026-06-19) + + +### Features + +* **backend:** auto-reconnect postgres trigger listener with backoff (WIN-2073) ([#9666](https://github.com/windmill-labs/windmill/issues/9666)) ([a425431](https://github.com/windmill-labs/windmill/commit/a425431e9067bcf85474fdc7b7ef7f73e41b9071)) + + +### Bug Fixes + +* **backend:** grant notify_event access to windmill roles ([#9665](https://github.com/windmill-labs/windmill/issues/9665)) ([a682d02](https://github.com/windmill-labs/windmill/commit/a682d02311a2110bfc0d5e0a5b52e96147fe0dd7)) +* **mcp:** repair invalid type keywords in tool JSON schemas ([#9667](https://github.com/windmill-labs/windmill/issues/9667)) ([c30bdec](https://github.com/windmill-labs/windmill/commit/c30bdecea77ff9b4d74d52961f3101201099b683)) +* trigger flow error handler on unrecoverable (OOM/zombie) step failures ([#9662](https://github.com/windmill-labs/windmill/issues/9662)) ([7e4df02](https://github.com/windmill-labs/windmill/commit/7e4df02bd60c4d6ee8c92d3dfd19f4e587ff9632)) + +## [1.730.0](https://github.com/windmill-labs/windmill/compare/v1.729.0...v1.730.0) (2026-06-18) + + +### Features + +* **ai-chat:** summary-based conversation compaction ([#9645](https://github.com/windmill-labs/windmill/issues/9645)) ([5d553b8](https://github.com/windmill-labs/windmill/commit/5d553b81c06664aab61131a93b198575c088d12d)) +* Data Pipelines alpha ([#9193](https://github.com/windmill-labs/windmill/issues/9193)) ([7155a0b](https://github.com/windmill-labs/windmill/commit/7155a0bb96cf30bd878272a0f4c3c3b02341b261)) + + +### Bug Fixes + +* **ai-chat:** stop echoing app draft value in global chat write tool results ([#9658](https://github.com/windmill-labs/windmill/issues/9658)) ([2fed808](https://github.com/windmill-labs/windmill/commit/2fed808b9e716d9a44b34c7a073ec0d37374be05)) +* **backend:** include raw_app drafts in list_apps draft_users ([#9647](https://github.com/windmill-labs/windmill/issues/9647)) ([19bc005](https://github.com/windmill-labs/windmill/commit/19bc0052f1069d732231950a0ec958f675d57417)) +* **frontend:** keep ?new_draft flag until first save is confirmed ([#9656](https://github.com/windmill-labs/windmill/issues/9656)) ([9b6b7c3](https://github.com/windmill-labs/windmill/commit/9b6b7c3862d9988e5e91eaab2b967a23f41cdc0d)) +* **frontend:** re-key raw-app autosave on post-deploy navigation ([#9646](https://github.com/windmill-labs/windmill/issues/9646)) ([1058bde](https://github.com/windmill-labs/windmill/commit/1058bdeccdc4c403ef4599db0ee74a65a66c715f)) +* gate agent-worker global setting reads with a blocklist ([#9623](https://github.com/windmill-labs/windmill/issues/9623)) ([fdd82f0](https://github.com/windmill-labs/windmill/commit/fdd82f0c48f29805cd9e219649f27fba45c7fd92)) +* **workspaces:** add instance setting to disable workspace invite/add emails ([#9643](https://github.com/windmill-labs/windmill/issues/9643)) ([796230d](https://github.com/windmill-labs/windmill/commit/796230d90a7e6d1debc15e139ab708881e527862)) + +## [1.729.0](https://github.com/windmill-labs/windmill/compare/v1.728.1...v1.729.0) (2026-06-18) + + +### Features + +* add ducklake schema support to the database manager ([#9633](https://github.com/windmill-labs/windmill/issues/9633)) ([3eeccaf](https://github.com/windmill-labs/windmill/commit/3eeccaf9682b7803fdf5be8dcbc4d243e0ba2e49)) +* **ai-chat:** self-hosted docs tools via windmill.dev llms.txt + ask benchmark ([#9578](https://github.com/windmill-labs/windmill/issues/9578)) ([f4425fc](https://github.com/windmill-labs/windmill/commit/f4425fca9fb0d02b845bd72888ade54905c5a30b)) +* **frontend:** View Diff and in-place Load for other users' drafts ([#9621](https://github.com/windmill-labs/windmill/issues/9621)) ([5508f1d](https://github.com/windmill-labs/windmill/commit/5508f1da9cd04c2583eb3f7ee6bce19d067f2227)) +* per-user draft review & deploy page (gating, badges, rename, raw-app deploy fixes) ([#9625](https://github.com/windmill-labs/windmill/issues/9625)) ([e09cd58](https://github.com/windmill-labs/windmill/commit/e09cd5862cb636e143027fe8d9a5be9c7097b031)) +* queue messages typed while ai chat is streaming ([#9525](https://github.com/windmill-labs/windmill/issues/9525)) ([51bd869](https://github.com/windmill-labs/windmill/commit/51bd8692a482850f7ac8b04dd16db5876336b5b9)) +* zero-setup oauth client credentials for registry providers ([#9559](https://github.com/windmill-labs/windmill/issues/9559)) ([e26a923](https://github.com/windmill-labs/windmill/commit/e26a9239a62a25abf90ef06ade4dde7f36e791bb)) + + +### Bug Fixes + +* **ai_evals:** adapt global eval harness to DB-backed user drafts ([#9641](https://github.com/windmill-labs/windmill/issues/9641)) ([e87ff79](https://github.com/windmill-labs/windmill/commit/e87ff79ecf6a6e0958916ed1b3756fb3addf719f)) +* **drafts:** preserve original timestamp when migrating localStorage drafts ([#9638](https://github.com/windmill-labs/windmill/issues/9638)) ([8021775](https://github.com/windmill-labs/windmill/commit/8021775f5f961ef6fd01b022639b85855326a1da)) +* **frontend:** don't save drafts on leave when auto-save is off, warn instead ([#9630](https://github.com/windmill-labs/windmill/issues/9630)) ([2523465](https://github.com/windmill-labs/windmill/commit/252346500945a9571af744c839ac0c7d6870504f)) +* **frontend:** render Modal2 dialogs above the AI chat panel ([#9636](https://github.com/windmill-labs/windmill/issues/9636)) ([b67c8cf](https://github.com/windmill-labs/windmill/commit/b67c8cf42b477575fc1bc448058ec0d3b7e54fee)) +* **frontend:** show AI sessions when AI unconfigured, with disabled chat ([#9644](https://github.com/windmill-labs/windmill/issues/9644)) ([ba69d81](https://github.com/windmill-labs/windmill/commit/ba69d8147b615e160cf3d2885fc65a0777b78b71)) +* **git-sync:** bump default sync script to hub/28719 (windmill-cli 1.728.1) for WAC modules ([#9649](https://github.com/windmill-labs/windmill/issues/9649)) ([3c0e38b](https://github.com/windmill-labs/windmill/commit/3c0e38b5890d77983cb5cf5f422a62a73e7a4f22)) + +## [1.728.1](https://github.com/windmill-labs/windmill/compare/v1.728.0...v1.728.1) (2026-06-17) + + +### Bug Fixes + +* **backend:** purge workspace_diff cache on workspace delete ([#9627](https://github.com/windmill-labs/windmill/issues/9627)) ([8a3f69d](https://github.com/windmill-labs/windmill/commit/8a3f69dda8f2088fb859ed8ed6e54458940423d0)) +* **cli:** fall back to esbuild-wasm on native host/binary mismatch ([#9629](https://github.com/windmill-labs/windmill/issues/9629)) ([86d1d16](https://github.com/windmill-labs/windmill/commit/86d1d160f0d3bd9faabdafada07e2956dd98445d)) +* **frontend:** persist session-editor draft path/summary edits + per-line diff tooltips ([#9622](https://github.com/windmill-labs/windmill/issues/9622)) ([e4bfeb2](https://github.com/windmill-labs/windmill/commit/e4bfeb29bc4e89669863b5f6396904a331167658)) + +## [1.728.0](https://github.com/windmill-labs/windmill/compare/v1.727.0...v1.728.0) (2026-06-16) + + +### Features + +* **frontend:** adapt AI-chat/sessions drafts to DB-backed model ([#9601](https://github.com/windmill-labs/windmill/issues/9601)) ([611c70a](https://github.com/windmill-labs/windmill/commit/611c70acd211cf4b8f8308da4a264c670a2f5f43)) +* **frontend:** consolidate draft-migration errors into a single toast + modal ([#9612](https://github.com/windmill-labs/windmill/issues/9612)) ([bc0d5bf](https://github.com/windmill-labs/windmill/commit/bc0d5bf241df3633921bd9d43d171e91034fbfcf)) +* **frontend:** dedup user drafts against the deployed baseline ([#9618](https://github.com/windmill-labs/windmill/issues/9618)) ([a2ce446](https://github.com/windmill-labs/windmill/commit/a2ce44645fdbfa98bf250fac2d15d2b5b26c4b47)) + + +### Bug Fixes + +* **frontend:** reset deleteWorkspaceForkModal on confirm in SidebarContent ([#9619](https://github.com/windmill-labs/windmill/issues/9619)) ([7cb5c6e](https://github.com/windmill-labs/windmill/commit/7cb5c6e749b2020dee5ee1499f0dc69c5109a6d8)) +* **frontend:** session Drafts drawer uses raw_app kind for the raw-app diff ([#9617](https://github.com/windmill-labs/windmill/issues/9617)) ([46288b6](https://github.com/windmill-labs/windmill/commit/46288b6143efae4dfdf6fe068b97a1e8831fce6a)) +* **nativets:** respect custom CA certs in in-process fetch runtime ([#9615](https://github.com/windmill-labs/windmill/issues/9615)) ([41562c7](https://github.com/windmill-labs/windmill/commit/41562c7d7c708d7d056d9b3d0c39b994a6f4a016)) +* **ResourceForm:** initialize JSON editor when resource type schema is unavailable ([#9611](https://github.com/windmill-labs/windmill/issues/9611)) ([5a24057](https://github.com/windmill-labs/windmill/commit/5a2405743b4622fc1021109114d007057abd5dfd)) +* show folder labels in the folder list table ([#9620](https://github.com/windmill-labs/windmill/issues/9620)) ([651fa13](https://github.com/windmill-labs/windmill/commit/651fa13ee80ff76e5a53ef1ed545b03ce6792294)) +* show last updated date per user in other-users-drafts modal ([#9614](https://github.com/windmill-labs/windmill/issues/9614)) ([f6104ce](https://github.com/windmill-labs/windmill/commit/f6104ce05c4005ffb9fe8112782d1ef6d3065300)) + ## [1.727.0](https://github.com/windmill-labs/windmill/compare/v1.726.1...v1.727.0) (2026-06-16) diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index 2bcc0abc04..b45df70011 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -16,6 +16,9 @@ export interface PromptRunResult { output: string; durationMs: number; tokenUsage: BenchmarkTokenUsage | null; + // Input tokens on the last assistant turn. The SDK `result` message reports + // usage cumulatively, so the final context size comes from per-turn usage. + finalContextTokens: number | null; trace: CliTrace; } @@ -144,6 +147,7 @@ export async function runPromptAndCapture( let output = ""; let assistantMessageCount = 0; let tokenUsage: BenchmarkTokenUsage | null = null; + let finalContextTokens: number | null = null; const startedAt = Date.now(); const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME); const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME); @@ -166,6 +170,12 @@ export async function runPromptAndCapture( for await (const message of query({ prompt, options })) { if (message.type === "assistant") { assistantMessageCount += 1; + const turnContext = anthropicUsageToBenchmarkTokenUsage( + message.message?.usage + )?.prompt; + if (turnContext && turnContext > 0) { + finalContextTokens = turnContext; + } const content = message.message?.content; if (Array.isArray(content)) { for (const block of content) { @@ -210,6 +220,7 @@ export async function runPromptAndCapture( output, durationMs: Date.now() - startedAt, tokenUsage, + finalContextTokens, trace: { toolsUsed, skillsInvoked, diff --git a/ai_evals/adapters/frontend/benchmarkRunner.ts b/ai_evals/adapters/frontend/benchmarkRunner.ts index 1729df7170..32107eadf1 100644 --- a/ai_evals/adapters/frontend/benchmarkRunner.ts +++ b/ai_evals/adapters/frontend/benchmarkRunner.ts @@ -96,7 +96,12 @@ async function getModeRunner( } function parseMode(value: string | undefined): FrontendBenchmarkMode { - if (value === "flow" || value === "app" || value === "script" || value === "global") { + if ( + value === "flow" || + value === "app" || + value === "script" || + value === "global" + ) { return value; } throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`); diff --git a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts index 16543b28de..fa52e6e826 100644 --- a/ai_evals/adapters/frontend/core/app/appEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/app/appEvalRunner.ts @@ -38,6 +38,7 @@ export interface AppEvalResult { toolCallCount: number; toolsUsed: string[]; tokenUsage: TokenUsage; + finalContextTokens: number | null; } export interface AppEvalOptions { @@ -113,6 +114,7 @@ export async function runAppEval( toolCallCount: rawResult.toolCallsCount, toolsUsed: rawResult.toolsCalled, tokenUsage: rawResult.tokenUsage, + finalContextTokens: rawResult.finalContextTokens, }; } finally { await cleanup(); diff --git a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts index 0cd25f5787..f533595ed2 100644 --- a/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/flow/flowEvalRunner.ts @@ -39,6 +39,7 @@ export interface FlowEvalResult { toolsUsed: string[]; toolCallDetails: ToolCallDetail[]; tokenUsage: TokenUsage; + finalContextTokens: number | null; } export interface FlowEvalOptions { @@ -113,6 +114,7 @@ export async function runFlowEval( toolsUsed: rawResult.toolsCalled, toolCallDetails: rawResult.toolCallDetails, tokenUsage: rawResult.tokenUsage, + finalContextTokens: rawResult.finalContextTokens, }; } finally { await cleanup(); diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index 058adc3644..a6d78da36b 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -9,6 +9,7 @@ import { } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; import { clearGlobalDrafts, + getGlobalDraft, listGlobalDrafts, } from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter"; import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared"; @@ -18,6 +19,7 @@ import type { GlobalDraftState } from "../../../../core/validators"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; import { registerBenchmarkWorkspaceRunnables, + seedBenchmarkDraft, unregisterBenchmarkWorkspaceRunnables, type BenchmarkWorkspaceRunnables, } from "../../mockBackend"; @@ -30,6 +32,10 @@ const MUTATING_GLOBAL_TOOLS = new Set([ ]); const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV = "WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT"; +// A/B gate for the search_app read tool: set to "1" to run the baseline arm +// (toolset without search_app) so its token cost can be compared against the arm +// that offers it. +const DISABLE_SEARCH_APP_ENV = "WMILL_AI_EVAL_DISABLE_SEARCH_APP"; const LIVE_EDITOR_ITEM_KINDS = { script: "script", @@ -53,6 +59,7 @@ export interface GlobalEvalResult { toolsUsed: string[]; toolCallDetails: ToolCallDetail[]; tokenUsage: TokenUsage; + finalContextTokens: number | null; } export interface GlobalEvalOptions { @@ -94,7 +101,7 @@ export async function runGlobalEval( tools: getGlobalEvalTools(), helpers: {}, apiKey, - getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }), + getOutput: () => collectGlobalDraftState(workspaceRoot), onAssistantMessageStart: options.runContext?.onAssistantMessageStart, onAssistantToken: options.runContext?.onAssistantChunk, onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, @@ -119,6 +126,7 @@ export async function runGlobalEval( toolsUsed: rawResult.toolsCalled, toolCallDetails: rawResult.toolCallDetails, tokenUsage: rawResult.tokenUsage, + finalContextTokens: rawResult.finalContextTokens, }; } finally { clearGlobalDrafts(workspaceRoot); @@ -130,6 +138,32 @@ export async function runGlobalEval( } } +// Build the harness output from the DB-backed drafts. `listGlobalDrafts` returns +// metadata-only rows for backend drafts (the model's `write_script` etc. persist +// straight to the backend with no in-tab editor cell), so re-read each such row +// with `getGlobalDraft` to attach the full value the validators assert on. A row +// that already carries a value (the production in-tab cell overlay) is kept as-is. +async function collectGlobalDraftState( + workspace: string, +): Promise { + const items = await listGlobalDrafts(workspace); + const drafts = await Promise.all( + items.map(async (item) => { + if (item.value !== undefined) { + return item; + } + const full = await getGlobalDraft( + workspace, + item.type, + item.path, + item.triggerKind, + ); + return full ?? item; + }), + ); + return { drafts: drafts as GlobalDraftState["drafts"] }; +} + function seedLiveEditorDrafts( workspace: string, fixtures: GlobalLiveEditorDraftFixture[], @@ -138,7 +172,9 @@ function seedLiveEditorDrafts( const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type]; const storagePath = fixture.storagePath ?? fixture.effectivePath ?? ""; if (fixture.value !== undefined) { - UserDraft.save(itemKind, storagePath, fixture.value, { workspace }); + // Seed as a backend draft row, not an in-tab cell: a cell would shadow the + // model's DB-backed edit when the output is read back via listGlobalDrafts. + seedBenchmarkDraft(workspace, itemKind, storagePath, fixture.value); } UserDraft.setLiveEditorDraft({ workspace, @@ -161,25 +197,28 @@ function clearLiveEditorDrafts( } function getGlobalEvalTools(): ProductionTool<{}>[] { - return (globalTools as ProductionTool<{}>[]).map((tool) => { - if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) { - return tool; - } + const disableSearchApp = process.env[DISABLE_SEARCH_APP_ENV] === "1"; + return (globalTools as ProductionTool<{}>[]) + .filter((tool) => !(disableSearchApp && tool.def.function.name === "search_app")) + .map((tool) => { + if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) { + return tool; + } - return { - ...tool, - requiresConfirmation: false, - validateBeforeConfirmation: undefined, - fn: async () => - JSON.stringify( - { - success: false, - error: - "This mutating workspace tool is disabled during ai_evals global mode.", - }, - null, - 2, - ), - }; - }); + return { + ...tool, + requiresConfirmation: false, + validateBeforeConfirmation: undefined, + fn: async () => + JSON.stringify( + { + success: false, + error: + "This mutating workspace tool is disabled during ai_evals global mode.", + }, + null, + 2, + ), + }; + }); } diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index 95ce6555e6..b9c80f3554 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -25,6 +25,7 @@ export interface ScriptEvalResult { toolsUsed: string[]; toolCallDetails: ToolCallDetail[]; tokenUsage: TokenUsage; + finalContextTokens: number | null; } export interface ScriptEvalOptions { @@ -111,6 +112,7 @@ export async function runScriptEval( toolsUsed: rawResult.toolsCalled, toolCallDetails: rawResult.toolCallDetails, tokenUsage: rawResult.tokenUsage, + finalContextTokens: rawResult.finalContextTokens, }; } finally { await cleanup(); diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 5f4ea2c307..f787743dfa 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -38,8 +38,9 @@ export interface RunEvalParams { helpers: THelpers; /** API key for the provider */ apiKey: string; - /** Function to get the current output state */ - getOutput: () => TOutput; + /** Function to get the current output state. May be async — global mode reads + * DB-backed drafts back through the (mocked) backend to build its output. */ + getOutput: () => TOutput | Promise; /** Model and Windmill backend configuration */ options: EvalRunnerOptions; onAssistantMessageStart?: () => void; @@ -154,9 +155,10 @@ export async function runEval( if (result.hitMaxIterations) { return { success: false, - output: getOutput(), + output: (await getOutput()) as TOutput, error: `Reached max turns (${maxIterations})`, tokenUsage: result.tokenUsage, + finalContextTokens: result.lastIterationUsage?.prompt ?? null, toolCallsCount, toolsCalled, toolCallDetails, @@ -170,8 +172,9 @@ export async function runEval( return { success: true, - output: getOutput(), + output: (await getOutput()) as TOutput, tokenUsage: result.tokenUsage, + finalContextTokens: result.lastIterationUsage?.prompt ?? null, toolCallsCount, toolsCalled, toolCallDetails, @@ -191,9 +194,10 @@ export async function runEval( return { success: false, - output: getOutput(), + output: (await getOutput()) as TOutput, error: errorMessage, tokenUsage: { prompt: 0, completion: 0, total: 0 }, + finalContextTokens: null, toolCallsCount, toolsCalled, toolCallDetails, diff --git a/ai_evals/adapters/frontend/core/shared/types.ts b/ai_evals/adapters/frontend/core/shared/types.ts index f081fe398a..c7af6c43e7 100644 --- a/ai_evals/adapters/frontend/core/shared/types.ts +++ b/ai_evals/adapters/frontend/core/shared/types.ts @@ -28,6 +28,8 @@ export interface RawEvalResult { output: TOutput; error?: string; tokenUsage: TokenUsage; + /** Input tokens on the last model request of the loop (see BenchmarkAttemptResult.finalContextTokens). */ + finalContextTokens: number | null; toolCallsCount: number; toolsCalled: string[]; toolCallDetails: ToolCallDetail[]; diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 622c79bf46..b78351512d 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -1,9 +1,20 @@ import { randomUUID } from 'node:crypto' -import type { CompletedJob, Flow, Job, Script } from '../../../frontend/src/lib/gen' +import type { + AppWithLastVersion, + CompletedJob, + Flow, + Job, + ListableApp, + Script +} from '../../../frontend/src/lib/gen' import type { DataTableTables, DataTableTableSchema, - ScriptLang + GetDraftForUserResponse, + ListDraftsResponse, + ScriptLang, + UpdateDraftResponse, + UserDraftItemKind } from '../../../frontend/src/lib/gen/types.gen' import { buildScriptLintResult } from './core/script/preview' import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine' @@ -29,6 +40,18 @@ export interface BenchmarkWorkspaceFlow { value: Flow['value'] } +export interface BenchmarkWorkspaceApp { + path: string + summary: string + value: { + files: Record + runnables: Record + data?: unknown + policy?: unknown + custom_path?: unknown + } +} + export interface BenchmarkWorkspaceJob { /** Stable id so a case prompt can reference a specific run (e.g. for get_job_logs). */ id?: string @@ -43,6 +66,7 @@ export interface BenchmarkWorkspaceJob { export interface BenchmarkWorkspaceRunnables { scripts?: BenchmarkWorkspaceScript[] flows?: BenchmarkWorkspaceFlow[] + apps?: BenchmarkWorkspaceApp[] datatables?: BenchmarkDatatableSeed[] jobs?: BenchmarkWorkspaceJob[] } @@ -63,6 +87,7 @@ export function resetBenchmarkMockBackend(): void { benchmarkWorkspaces.clear() benchmarkWorkspaceRunnables.clear() benchmarkJobs.clear() + benchmarkDrafts.clear() } export function registerBenchmarkWorkspace(workspace: string): void { @@ -74,6 +99,8 @@ export function registerBenchmarkWorkspaceRunnables( runnables: BenchmarkWorkspaceRunnables ): void { benchmarkWorkspaces.add(workspace) + // Fresh case: drop any drafts left from a prior run on this workspace id. + clearBenchmarkDrafts(workspace) // Datatables are mutated in place by exec_datatable_sql (a write must be visible // to later reads), so store an isolated deep copy — never mutate the caller's seed. benchmarkWorkspaceRunnables.set(workspace, { @@ -98,6 +125,7 @@ export function registerBenchmarkWorkspaceRunnables( export function unregisterBenchmarkWorkspace(workspace: string): void { benchmarkWorkspaces.delete(workspace) benchmarkWorkspaceRunnables.delete(workspace) + clearBenchmarkDrafts(workspace) for (const [jobId, entry] of benchmarkJobs.entries()) { if (entry.workspace === workspace) { benchmarkJobs.delete(jobId) @@ -153,6 +181,22 @@ export function getBenchmarkFlowByPath(workspace: string, path: string): Flow | return flow ? buildBenchmarkFlow(flow) : null } +export function listBenchmarkApps(workspace: string): ListableApp[] | null { + const runnables = benchmarkWorkspaceRunnables.get(workspace) + if (!runnables) { + return null + } + return (runnables.apps ?? []).map(buildBenchmarkListableApp) +} + +export function getBenchmarkAppByPath(workspace: string, path: string): AppWithLastVersion | null { + const app = benchmarkWorkspaceRunnables + .get(workspace) + ?.apps?.find((entry) => entry.path === path) + + return app ? buildBenchmarkApp(app) : null +} + export function createBenchmarkCompletedJob(input: { workspace: string jobKind: CompletedJob['job_kind'] @@ -238,6 +282,110 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { return job.logs ?? '' } +// ============= Drafts (per-user, DB-backed in production) ============= + +/** + * In-memory stand-in for the per-user draft backend (`DraftService`). The global + * AI chat now persists and reads drafts through the backend DB instead of an + * in-tab `UserDraft` cell, so the eval mocks the three draft endpoints it + * exercises (`updateDraft` / `getDraftForUser` / `listDrafts`) and keeps the + * saved values here, keyed by workspace + draft kind + storage path. Mirrors the + * semantics of the production unit test's mock in + * `frontend/src/lib/components/copilot/chat/global/core.test.ts`. + */ +const benchmarkDrafts = new Map< + string, + { workspace: string; kind: UserDraftItemKind; path: string; value: unknown } +>() + +// Fixed timestamp so artifacts stay deterministic. No eval simulates a +// concurrent writer, so every save is accepted and the conflict branch is +// never taken — the syncer just records this as its `last_sync` baseline. +const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z' + +function benchmarkDraftKey(workspace: string, kind: string, path: string): string { + return `${workspace}::${kind}::${path}` +} + +export function clearBenchmarkDrafts(workspace: string): void { + for (const [key, entry] of benchmarkDrafts.entries()) { + if (entry.workspace === workspace) { + benchmarkDrafts.delete(key) + } + } +} + +/** + * Seed a draft straight into the store — used by the eval's live-editor draft + * fixtures, which model "the user already has this draft open/saved". Writing it + * here (instead of through `UserDraft.save`) keeps it a backend draft row with no + * shadowing in-tab cell, so a model edit that persists to the backend is what the + * output read-back captures — not the stale seed. + */ +export function seedBenchmarkDraft( + workspace: string, + kind: UserDraftItemKind, + path: string, + value: unknown +): void { + benchmarkDrafts.set(benchmarkDraftKey(workspace, kind, path), { + workspace, + kind, + path, + value + }) +} + +/** Mirror `DraftService.updateDraft`: a `null`/omitted value deletes the row. */ +export function updateBenchmarkDraft(input: { + workspace: string + kind: UserDraftItemKind + path: string + requestBody?: { value?: unknown } +}): UpdateDraftResponse { + const key = benchmarkDraftKey(input.workspace, input.kind, input.path) + const value = input.requestBody?.value + if (value == null) { + benchmarkDrafts.delete(key) + } else { + benchmarkDrafts.set(key, { + workspace: input.workspace, + kind: input.kind, + path: input.path, + value + }) + } + return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP } +} + +/** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the + * adapter's narrowed catch treats it as "no draft" instead of re-throwing. */ +export function getBenchmarkDraftForUser(input: { + workspace: string + kind: UserDraftItemKind + path: string +}): GetDraftForUserResponse { + const entry = benchmarkDrafts.get(benchmarkDraftKey(input.workspace, input.kind, input.path)) + if (!entry) { + throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 }) + } + return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } +} + +/** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ +export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { + return [...benchmarkDrafts.values()] + .filter((entry) => entry.workspace === workspace) + .map((entry) => ({ + kind: entry.kind, + path: entry.path, + summary: (entry.value as { summary?: string } | null)?.summary, + draft_only: true, + legacy_draft: false, + created_at: BENCHMARK_DRAFT_TIMESTAMP + })) +} + // ============= Datatables (best-effort in-memory SQL) ============= /** @@ -492,3 +640,35 @@ function buildBenchmarkFlow(flow: BenchmarkWorkspaceFlow): Flow { extra_perms: {} } as Flow } + +function buildBenchmarkListableApp(app: BenchmarkWorkspaceApp): ListableApp { + return { + id: 0, + workspace_id: 'benchmark', + path: app.path, + summary: app.summary, + version: 1, + extra_perms: {}, + edited_at: BENCHMARK_TIMESTAMP, + execution_mode: 'viewer', + raw_app: true + } +} + +function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { + return { + id: 0, + workspace_id: 'benchmark', + path: app.path, + summary: app.summary, + versions: [1], + created_by: 'benchmark', + created_at: BENCHMARK_TIMESTAMP, + value: app.value, + policy: (app.value.policy ?? {}) as AppWithLastVersion['policy'], + execution_mode: 'viewer', + extra_perms: {}, + custom_path: app.value.custom_path as string | undefined, + raw_app: true + } +} diff --git a/ai_evals/adapters/frontend/mockBackendDrafts.test.ts b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts new file mode 100644 index 0000000000..a720de5e43 --- /dev/null +++ b/ai_evals/adapters/frontend/mockBackendDrafts.test.ts @@ -0,0 +1,94 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { + clearBenchmarkDrafts, + getBenchmarkDraftForUser, + listBenchmarkDrafts, + resetBenchmarkMockBackend, + seedBenchmarkDraft, + updateBenchmarkDraft +} from './mockBackend' + +const WORKSPACE = 'benchmark-drafts-ws' + +// Drives the in-memory stand-in for the per-user draft backend (`DraftService`) +// that the global AI-chat eval round-trips its drafts through. Mirrors the +// production-unit-test mock in +// `frontend/src/lib/components/copilot/chat/global/core.test.ts`. +describe('mockBackend drafts', () => { + beforeEach(() => resetBenchmarkMockBackend()) + afterEach(() => resetBenchmarkMockBackend()) + + it('round-trips a saved draft through update / get / list', () => { + const value = { summary: 'Greet a user', content: 'export async function main() {}' } + const res = updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'script', + path: 'f/evals/greet', + requestBody: { value } + }) + expect(res.status).toBe('saved') + + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/greet' }).value).toEqual( + value + ) + + const rows = listBenchmarkDrafts(WORKSPACE) + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ kind: 'script', path: 'f/evals/greet', summary: 'Greet a user', draft_only: true }) + }) + + it('treats a null value as a delete', () => { + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'variable', + path: 'f/evals/token', + requestBody: { value: { summary: 'token' } } + }) + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'variable', + path: 'f/evals/token', + requestBody: { value: null } + }) + + expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) + expect(() => getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'variable', path: 'f/evals/token' })).toThrow() + }) + + it('throws a 404-shaped error when no draft exists', () => { + try { + getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/missing' }) + throw new Error('expected a throw') + } catch (e) { + expect((e as { status?: number }).status).toBe(404) + } + }) + + it('seeds a draft as a backend row that a later edit overwrites', () => { + seedBenchmarkDraft(WORKSPACE, 'script', 'f/evals/current', { content: 'seed' }) + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ + content: 'seed' + }) + + // A model edit persists the same path and must win over the seed. + updateBenchmarkDraft({ + workspace: WORKSPACE, + kind: 'script', + path: 'f/evals/current', + requestBody: { value: { content: 'edited' } } + }) + expect(getBenchmarkDraftForUser({ workspace: WORKSPACE, kind: 'script', path: 'f/evals/current' }).value).toEqual({ + content: 'edited' + }) + }) + + it('clears only the targeted workspace', () => { + seedBenchmarkDraft(WORKSPACE, 'script', 'f/a', { content: 'a' }) + seedBenchmarkDraft('other-ws', 'script', 'f/b', { content: 'b' }) + + clearBenchmarkDrafts(WORKSPACE) + + expect(listBenchmarkDrafts(WORKSPACE)).toHaveLength(0) + expect(listBenchmarkDrafts('other-ws')).toHaveLength(1) + }) +}) diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index ebbbac8d11..621ecaefcd 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -33,15 +33,19 @@ vi.mock('$lib/components/vscode', () => ({})) vi.mock('$lib/gen', async () => { const actual = await vi.importActual('$lib/gen') const { + getBenchmarkAppByPath, getBenchmarkCompletedJob, getBenchmarkCompletedJobResultMaybe, getBenchmarkDatatableSchema, + getBenchmarkDraftForUser, getBenchmarkFlowByPath, getBenchmarkJobLogs, getBenchmarkScriptByHash, getBenchmarkScriptByPath, hasBenchmarkWorkspace, + listBenchmarkApps, listBenchmarkDatatables, + listBenchmarkDrafts, listBenchmarkFlows, listBenchmarkJobs, listBenchmarkScripts, @@ -50,7 +54,8 @@ vi.mock('$lib/gen', async () => { previewBenchmarkSchedule, runBenchmarkDatatableSql, runBenchmarkFlowByPath, - runBenchmarkScriptPreview + runBenchmarkScriptPreview, + updateBenchmarkDraft } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -66,6 +71,25 @@ vi.mock('$lib/gen', async () => { return { ...actual, + DraftService: wrapService(actual.DraftService, { + updateDraft: async (data: { + workspace: string + kind: any + path: string + requestBody?: { value?: unknown } + }) => + hasBenchmarkWorkspace(data.workspace) + ? updateBenchmarkDraft(data) + : actual.DraftService.updateDraft(data), + getDraftForUser: async (data: { workspace: string; kind: any; path: string }) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkDraftForUser(data) + : actual.DraftService.getDraftForUser(data), + listDrafts: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? listBenchmarkDrafts(data.workspace) + : actual.DraftService.listDrafts(data) + }), ScriptService: wrapService(actual.ScriptService, { listScripts: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) @@ -277,12 +301,20 @@ vi.mock('$lib/gen', async () => { }), AppService: wrapService(actual.AppService, { existsApp: async (data: { workspace: string; path: string }) => - hasBenchmarkWorkspace(data.workspace) ? false : actual.AppService.existsApp(data), + hasBenchmarkWorkspace(data.workspace) + ? Boolean(getBenchmarkAppByPath(data.workspace, data.path)) + : actual.AppService.existsApp(data), listApps: async (data: { workspace: string }) => - hasBenchmarkWorkspace(data.workspace) ? [] : actual.AppService.listApps(data), + hasBenchmarkWorkspace(data.workspace) + ? (listBenchmarkApps(data.workspace) ?? []) + : actual.AppService.listApps(data), getAppByPath: async (data: { workspace: string; path: string }) => { if (hasBenchmarkWorkspace(data.workspace)) { - throw new Error(`App "${data.path}" not found in benchmark workspace`) + const app = getBenchmarkAppByPath(data.workspace, data.path) + if (!app) { + throw new Error(`App "${data.path}" not found in benchmark workspace`) + } + return app } return actual.AppService.getAppByPath(data) } @@ -434,5 +466,6 @@ benchmarkIt( resetBenchmarkMockBackend() } }, - 600_000 + // Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes. + 7_200_000 ) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 28839b0594..319d13b272 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -870,3 +870,246 @@ judgeChecklist: - fetches the logs for the requested job id - explains the failure from the returned logs (connection refused to the upstream API) + +# --- Documentation search (search_docs) --- +# Pure product-knowledge questions: the assistant should consult the docs via +# search_docs and answer conversationally, not draft or mutate anything. No +# draft is produced, so the global judge is skipped and we validate tool use. + +- id: global-docs-ai-agent-step + prompt: |- + Does Windmill support a flow step where an LLM decides which of my scripts to call based on the input? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-retry-step + prompt: |- + How does automatic retry work for a flow step that calls a flaky API? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-key-value-store + prompt: |- + Can I use a Redis-style key-value store from my Windmill scripts, and how? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +- id: global-docs-cron-schedule-format + prompt: |- + How do Windmill's cron schedules work, and what format does the schedule expression use? + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_docs + forbiddenToolsUsed: + - write_script + - write_flow + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + +# --- Raw app on a large project (context-usage benchmark) --- +# These cases run against the deliberately large `analytics_dashboard` raw-app +# fixture (~20 frontend files incl. a 5k-line data module, plus backend runnables). +# They exist to measure how much context the global chat consumes when working in a +# big raw app: test29 is a read-heavy debugging hunt, test30 is a small edit baseline. +# tokenUsage is recorded per run, so the same cases re-run after a read-tool change +# (the read_app_file cap + offset/limit paging) quantify the optimization. skipJudge: +# the judge only sees the drafts artifact and cannot run the app, so we validate +# deterministically. + +- id: global-test29-raw-app-debug-large + prompt: |- + The analytics dashboard app at `f/evals/global/analytics_dashboard` has a bug: + the Revenue Summary tile shows a total that is lower than the per-order line + totals and the per-region breakdown. Track down what is computing revenue + incorrectly and fix it. Keep the change as an AI draft only; do not deploy or + save it. + initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard + runtime: + maxTurns: 20 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/analytics_dashboard + valueIncludes: + - "return order.unitPrice * order.quantity" + toolExpect: + requiredToolsAnyOf: + # Inspecting the app's files is satisfied by either reading them directly + # or grepping for the revenue calculation. + - [read_app_file, search_app] + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - inspects the dashboard app's files to locate the revenue calculation + - fixes the per-order revenue so it multiplies unit price by quantity + - leaves the result as an AI draft and does not deploy or save it + +- id: global-test30-raw-app-small-edit-large + prompt: |- + In the dashboard app at `f/evals/global/analytics_dashboard`, change the main + page heading from "Operations Console" to "Revenue Overview". Leave everything + else unchanged. Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/analytics_dashboard + valueIncludes: + - "Revenue Overview" + toolExpect: + requiredToolsAnyOf: + # Inspecting the app's files is satisfied by reading them directly or + # grepping for the target with search_app. + - [read_app_file, search_app] + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - renames the main page heading to Revenue Overview + - does not change other dashboard behavior + - leaves the result as an AI draft only + +- id: global-test31-raw-app-debug-inspect-data + prompt: |- + The raw app dashboard at `f/evals/global/analytics_dashboard` is reporting + revenue totals that look too low. Inspect the app's files — both the sample + order data module and the revenue calculation — to work out whether the bug is + in the data or in the calculation, then fix the actual cause. Keep the change as + an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard + runtime: + maxTurns: 22 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/analytics_dashboard + valueIncludes: + - "return order.unitPrice * order.quantity" + toolExpect: + requiredToolsAnyOf: + # Inspecting the app's files is satisfied by reading them directly or + # grepping for the target with search_app. + - [read_app_file, search_app] + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - inspects both the sample order data module and the revenue aggregation logic + - identifies the per-order revenue bug and fixes it to multiply unit price by quantity + - leaves the result as an AI draft only + +- id: global-test32-raw-app-cross-file-consistency + prompt: |- + The raw app dashboard at `f/evals/global/analytics_dashboard` shows revenue + totals that disagree between the Revenue Summary tile, the orders table, and the + regional breakdown. Investigate how each of those computes revenue, work out + which calculation is wrong, and fix it. Keep the change as an AI draft only; do + not deploy or save it. + # Cross-file investigation: forces the model through several overlapping files + # (the summary's aggregation helper, the orders table, the regional breakdown) — + # a realistic multi-file read load that exercises the read_app_file cap. + initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard + runtime: + maxTurns: 24 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/analytics_dashboard + valueIncludes: + - "return order.unitPrice * order.quantity" + toolExpect: + requiredToolsAnyOf: + # Inspecting the app's files is satisfied by reading them directly or + # grepping for the target with search_app. + - [read_app_file, search_app] + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - inspects the revenue calculation behind the summary tile, the orders table, and the regional breakdown + - identifies that the per-order revenue helper omits quantity and fixes it to multiply unit price by quantity + - leaves the result as an AI draft only + +- id: global-test33-raw-app-rename-across-files + prompt: |- + In the dashboard app at `f/evals/global/analytics_dashboard`, rename the + `formatCurrency` helper to `formatMoney` everywhere it is defined, imported, and + called. Leave the separate `formatCurrencyPrecise` helper exactly as it is. Keep + the change as an AI draft only; do not deploy or save it. + # Find-all-usages rename: formatCurrency is defined once and called in 6 places + # spread across 4 component files (and imported in 4). Locating every usage is the + # exact task search_app is meant to make cheap — one grep returns all file:line + # rows instead of reading each component whole. valueExcludes "formatCurrency(" + # asserts the definition and all call sites were renamed while tolerating the + # preserved formatCurrencyPrecise (which is never followed by "("). + initial: ai_evals/fixtures/frontend/global/initial/analytics_dashboard + runtime: + maxTurns: 22 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + path: f/evals/global/analytics_dashboard + valueIncludes: + - "export function formatMoney" + - "formatMoney(" + valueExcludes: + - "formatCurrency(" + toolExpect: + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - renames the formatCurrency definition, imports, and all call sites to formatMoney + - leaves the unrelated formatCurrencyPrecise helper unchanged + - leaves the result as an AI draft only diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 05e2f1527b..9955a73fa9 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -246,6 +246,21 @@ describe("loadCases", () => { }); }); + it("loads global docs-search cases as tool-use checks", async () => { + const globalCases = await loadCases("global"); + const docsCases = globalCases.filter((entry) => + entry.id.startsWith("global-docs-"), + ); + expect(docsCases.length).toBeGreaterThanOrEqual(3); + + // Each docs case verifies the assistant reaches for search_docs and does not + // draft anything; with no draft, the global judge is skipped. + for (const entry of docsCases) { + expect(entry.skipJudge).toBe(true); + expect(entry.toolExpect?.requiredToolsUsed).toContain("search_docs"); + } + }); + it("loads tool expectations for workspace mutation cases", async () => { const scriptCases = await loadCases("script"); const caseEntry = scriptCases.find( diff --git a/ai_evals/core/results.test.ts b/ai_evals/core/results.test.ts index 2d6077c5bd..a5406f987f 100644 --- a/ai_evals/core/results.test.ts +++ b/ai_evals/core/results.test.ts @@ -92,6 +92,71 @@ describe("benchmark results", () => { expect(summary).toContain("Average duration (all attempts): 550ms"); }); + it("aggregates final context size over passed attempts only", () => { + const result = buildRunResult({ + mode: "global", + runs: 1, + runModel: "model-under-test", + judgeModel: "judge-model", + caseResults: [ + caseResult([ + { + attempt: 1, + passed: true, + durationMs: 1000, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: { prompt: 12000, completion: 200, total: 12200 }, + finalContextTokens: 5000, + }, + { + attempt: 2, + passed: true, + durationMs: 1100, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["edit_script"], + skillsInvoked: [], + checks: [{ name: "edited", passed: true }], + judgeScore: 100, + judgeSummary: "ok", + error: null, + tokenUsage: { prompt: 18000, completion: 300, total: 18300 }, + finalContextTokens: 7000, + }, + { + attempt: 3, + passed: false, + durationMs: 100, + assistantMessageCount: 1, + toolCallCount: 0, + toolsUsed: [], + skillsInvoked: [], + checks: [{ name: "edited", passed: false }], + judgeScore: 10, + judgeSummary: "missed", + error: "failed", + tokenUsage: { prompt: 20000, completion: 100, total: 20100 }, + finalContextTokens: 9000, + }, + ]), + ], + }); + + // Final context size stays below cumulative prompt and ignores the failed attempt. + expect(result.averageFinalContextTokensPassed).toBe(6000); + expect(result.maxFinalContextTokensPassed).toBe(7000); + expect(formatRunSummary(result)).toContain( + "Final context size (passed): 6000 tokens (max 7000)", + ); + }); + it("reports passed averages as unavailable when no attempt passes", () => { const result = buildRunResult({ mode: "global", diff --git a/ai_evals/core/results.ts b/ai_evals/core/results.ts index 0b84497165..a9807d7d16 100644 --- a/ai_evals/core/results.ts +++ b/ai_evals/core/results.ts @@ -16,6 +16,9 @@ type AttemptAggregate = { durationTotal: number; tokenUsageAttemptCount: number; tokenUsageTotal: BenchmarkTokenUsage | null; + finalContextAttemptCount: number; + finalContextTotal: number; + finalContextMax: number | null; }; export async function writeRunResult( @@ -117,6 +120,8 @@ export function buildRunResult(input: { passedAttemptAggregate, passedAttempts, ), + averageFinalContextTokensPassed: averageFinalContext(passedAttemptAggregate), + maxFinalContextTokensPassed: passedAttemptAggregate.finalContextMax, cases: input.caseResults, }; } @@ -133,6 +138,11 @@ export function formatRunSummary(result: BenchmarkRunResult): string { `Average tokens (passed): ${formatTokenUsage(result.averageTokenUsagePerPassedAttempt)}`, ); } + if (result.averageFinalContextTokensPassed != null) { + lines.push( + `Final context size (passed): ${Math.round(result.averageFinalContextTokensPassed)} tokens (max ${Math.round(result.maxFinalContextTokensPassed ?? 0)})`, + ); + } if (result.passedAttempts < result.attemptCount) { lines.push( `Average duration (all attempts): ${Math.round(result.averageDurationMs)}ms`, @@ -181,10 +191,21 @@ function aggregateAttempts(attempts: BenchmarkAttemptResult[]): AttemptAggregate durationTotal: 0, tokenUsageAttemptCount: 0, tokenUsageTotal: null, + finalContextAttemptCount: 0, + finalContextTotal: 0, + finalContextMax: null, }; for (const attempt of attempts) { aggregate.durationTotal += attempt.durationMs; + if (typeof attempt.finalContextTokens === "number") { + aggregate.finalContextAttemptCount += 1; + aggregate.finalContextTotal += attempt.finalContextTokens; + aggregate.finalContextMax = Math.max( + aggregate.finalContextMax ?? 0, + attempt.finalContextTokens, + ); + } if (!attempt.tokenUsage) { continue; } @@ -204,6 +225,12 @@ function averageDuration(aggregate: AttemptAggregate): number | null { : aggregate.durationTotal / aggregate.attemptCount; } +function averageFinalContext(aggregate: AttemptAggregate): number | null { + return aggregate.finalContextAttemptCount === 0 + ? null + : aggregate.finalContextTotal / aggregate.finalContextAttemptCount; +} + function averageTokenUsage( aggregate: AttemptAggregate, denominator: number, @@ -318,6 +345,9 @@ function toHistoryRecord(result: BenchmarkRunResult) { averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null, averageTokenUsagePerPassedAttempt: result.averageTokenUsagePerPassedAttempt ?? null, + averageFinalContextTokensPassed: + result.averageFinalContextTokensPassed ?? null, + maxFinalContextTokensPassed: result.maxFinalContextTokensPassed ?? null, failedCaseIds: Array.from( new Set( result.cases @@ -361,6 +391,10 @@ function toHistoryRecord(result: BenchmarkRunResult) { passedAttemptAggregate, passedAttempts, ), + averageFinalContextTokensPassed: averageFinalContext( + passedAttemptAggregate, + ), + maxFinalContextTokensPassed: passedAttemptAggregate.finalContextMax, }; }), }; diff --git a/ai_evals/core/runSuite.ts b/ai_evals/core/runSuite.ts index ed82d841cb..1438749a11 100644 --- a/ai_evals/core/runSuite.ts +++ b/ai_evals/core/runSuite.ts @@ -225,7 +225,9 @@ async function runCaseAttempts(input: { checklist: input.evalCase.judgeChecklist, initial, expected: input.modeRunner.mode === "cli" ? undefined : expected, - actual: run.actual, + actual: input.modeRunner.prepareJudgeActual + ? input.modeRunner.prepareJudgeActual(run.actual) + : run.actual, model: input.judgeModel, }); @@ -255,6 +257,7 @@ async function runCaseAttempts(input: { judgeSummary, error: run.error ?? null, tokenUsage: run.tokenUsage ?? null, + finalContextTokens: run.finalContextTokens ?? null, artifactsPath: null, artifactFiles, }; @@ -291,6 +294,7 @@ async function runCaseAttempts(input: { judgeSummary: null, error: message, tokenUsage: null, + finalContextTokens: null, }; if (surface) { input.onProgress?.({ diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 27c2fcddac..52667fa321 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -168,11 +168,21 @@ export interface ToolCallArgumentRule { export interface ToolValidationSpec { requiredToolsUsed?: string[]; + /** + * Each inner array is an alternatives group: the check passes when at least + * one tool in the group was used. Use when several tools satisfy the same + * intent so a model that picks any valid path passes — e.g. inspecting an + * app's files via either `read_app_file` or `search_app`. + */ + requiredToolsAnyOf?: string[][]; forbiddenToolsUsed?: string[]; toolCallArgs?: ToolCallArgumentRule[]; } -export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec; +export type EvalValidationSpec = + | FlowValidationSpec + | AppValidationSpec + | GlobalValidationSpec; export interface EvalCase { id: string; @@ -249,6 +259,12 @@ export interface ModeRunOutput { toolCallDetails?: ToolCallDetail[]; skillsInvoked: string[]; tokenUsage?: BenchmarkTokenUsage | null; + /** + * Total input tokens occupying the context window on the LAST model request + * of the agentic loop (input + cache-creation + cache-read). Complements the + * cumulative `tokenUsage.prompt`, which sums every iteration's input. + */ + finalContextTokens?: number | null; } export interface ModeRunContext { @@ -294,6 +310,12 @@ export interface ModeRunner { context: ModeRunContext; }): Promise; buildArtifacts?(actual: TActual): BenchmarkArtifactFile[]; + /** + * Optional transform applied to `actual` before it is handed to the LLM judge. + * Use it to strip fields the judge must stay blind to (e.g. which docs-tool + * arm produced an answer). When omitted, the judge receives `actual` as-is. + */ + prepareJudgeActual?(actual: TActual): unknown; } export interface BenchmarkAttemptResult { @@ -310,6 +332,7 @@ export interface BenchmarkAttemptResult { judgeSummary: string | null; error: string | null; tokenUsage?: BenchmarkTokenUsage | null; + finalContextTokens?: number | null; artifactsPath?: string | null; artifactFiles?: BenchmarkArtifactFile[]; } @@ -340,6 +363,8 @@ export interface BenchmarkRunResult { totalPassedTokenUsage?: BenchmarkTokenUsage | null; averageTokenUsagePerAttempt?: BenchmarkTokenUsage | null; averageTokenUsagePerPassedAttempt?: BenchmarkTokenUsage | null; + averageFinalContextTokensPassed?: number | null; + maxFinalContextTokensPassed?: number | null; artifactsPath?: string | null; cases: BenchmarkCaseResult[]; } diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 7f0e841366..5183a20226 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -245,6 +245,49 @@ describe("validateToolExpectations", () => { 'accepted substrings: insert into, update; values: "DROP TABLE orders"', }); }); + + it("passes requiredToolsAnyOf when any alternative in the group is used", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["search_app", "patch_app_file"], + skillsInvoked: [], + }, + toolExpect: { + requiredToolsAnyOf: [["read_app_file", "search_app"]], + }, + }); + + expect(checks).toContainEqual({ + name: "uses one of read_app_file, search_app", + passed: true, + }); + }); + + it("fails requiredToolsAnyOf when no alternative in the group is used", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["patch_app_file"], + skillsInvoked: [], + }, + toolExpect: { + requiredToolsAnyOf: [["read_app_file", "search_app"]], + }, + }); + + expect(checks).toContainEqual({ + name: "uses one of read_app_file, search_app", + passed: false, + details: "tools used: patch_app_file", + }); + }); }); describe("validateGlobalState", () => { diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index e7a7641c00..23a6709f9b 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -169,6 +169,16 @@ export function validateToolExpectations(input: { ); } + for (const group of expect.requiredToolsAnyOf ?? []) { + checks.push( + check( + `uses one of ${group.join(", ")}`, + group.some((toolName) => input.run.toolsUsed.includes(toolName)), + `tools used: ${input.run.toolsUsed.join(", ") || "none"}` + ) + ); + } + for (const toolName of expect.forbiddenToolsUsed ?? []) { checks.push( check( diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/main.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/main.ts new file mode 100644 index 0000000000..b3c67cfdc7 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/main.ts @@ -0,0 +1,68 @@ +type OrderStatus = 'paid' | 'shipped' | 'delivered' | 'pending' | 'refunded' | 'cancelled' + +interface Order { + id: string + region: string + quantity: number + unitPrice: number + status: OrderStatus + placedAt: string +} + +// Server-side revenue rollup. Mirrors the client aggregation but is computed +// from the authoritative mocked order book so it can be used to cross-check +// the dashboard and to back the export. +const orders: Order[] = [ + { id: 'ORD-10001', region: 'North America', quantity: 3, unitPrice: 1195, status: 'delivered', placedAt: '2024-05-02' }, + { id: 'ORD-10002', region: 'EMEA', quantity: 5, unitPrice: 880, status: 'shipped', placedAt: '2024-05-03' }, + { id: 'ORD-10003', region: 'APAC', quantity: 2, unitPrice: 640, status: 'paid', placedAt: '2024-05-05' }, + { id: 'ORD-10004', region: 'LATAM', quantity: 7, unitPrice: 315, status: 'delivered', placedAt: '2024-05-07' }, + { id: 'ORD-10005', region: 'North America', quantity: 4, unitPrice: 150, status: 'refunded', placedAt: '2024-05-09' }, + { id: 'ORD-10006', region: 'EMEA', quantity: 6, unitPrice: 220, status: 'shipped', placedAt: '2024-05-12' }, + { id: 'ORD-10007', region: 'APAC', quantity: 1, unitPrice: 980, status: 'pending', placedAt: '2024-05-15' }, + { id: 'ORD-10008', region: 'North America', quantity: 8, unitPrice: 1100, status: 'delivered', placedAt: '2024-05-18' }, + { id: 'ORD-10009', region: 'EMEA', quantity: 2, unitPrice: 860, status: 'cancelled', placedAt: '2024-05-22' }, + { id: 'ORD-10010', region: 'LATAM', quantity: 9, unitPrice: 290, status: 'paid', placedAt: '2024-05-26' } +] + +const REVENUE_STATUSES: OrderStatus[] = ['paid', 'shipped', 'delivered'] + +export async function main({ + from, + to, + region +}: { + from: string + to: string + region: string +}): Promise<{ + totalRevenue: number + netRevenue: number + totalOrders: number + averageOrderValue: number + unitsSold: number + refundedRevenue: number + currency: string +}> { + let scoped = orders.filter((order) => order.placedAt >= from && order.placedAt <= to) + if (region && region !== 'all') { + scoped = scoped.filter((order) => order.region === region) + } + + const booked = scoped.filter((order) => REVENUE_STATUSES.includes(order.status)) + const totalRevenue = booked.reduce((acc, order) => acc + order.unitPrice * order.quantity, 0) + const unitsSold = booked.reduce((acc, order) => acc + order.quantity, 0) + const refundedRevenue = scoped + .filter((order) => order.status === 'refunded') + .reduce((acc, order) => acc + order.unitPrice * order.quantity, 0) + + return { + totalRevenue, + netRevenue: totalRevenue - refundedRevenue, + totalOrders: booked.length, + averageOrderValue: booked.length === 0 ? 0 : Math.round(totalRevenue / booked.length), + unitsSold, + refundedRevenue, + currency: 'USD' + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/meta.json b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/meta.json new file mode 100644 index 0000000000..ab2e5537f8 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/computeSummary/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Compute Summary", + "language": "bun" +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/main.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/main.ts new file mode 100644 index 0000000000..ec10c1d951 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/main.ts @@ -0,0 +1,51 @@ +// Builds a downloadable report for the current dashboard view. Returns a data +// URL the browser can open directly so the export works without object storage. +export async function main({ + from, + to, + region, + format +}: { + from: string + to: string + region: string + format: 'csv' | 'json' +}): Promise<{ url: string; rows: number; filename: string }> { + const summary = { + from, + to, + region: region || 'all', + generatedAt: new Date().toISOString(), + rows: [ + { region: 'North America', revenue: 211_400, orders: 168 }, + { region: 'EMEA', revenue: 142_900, orders: 121 }, + { region: 'APAC', revenue: 86_500, orders: 78 }, + { region: 'LATAM', revenue: 41_500, orders: 45 } + ] + } + + const scoped = + region && region !== 'all' + ? summary.rows.filter((row) => row.region === region) + : summary.rows + + let body: string + let mime: string + if (format === 'csv') { + const header = 'region,revenue,orders' + const lines = scoped.map((row) => `${row.region},${row.revenue},${row.orders}`) + body = [header, ...lines].join('\n') + mime = 'text/csv' + } else { + body = JSON.stringify({ ...summary, rows: scoped }, null, 2) + mime = 'application/json' + } + + const encoded = Buffer.from(body, 'utf-8').toString('base64') + const filename = `revenue-report-${from}_${to}.${format}` + return { + url: `data:${mime};base64,${encoded}`, + rows: scoped.length, + filename + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/meta.json b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/meta.json new file mode 100644 index 0000000000..8f198d716f --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/exportReport/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Export Report", + "language": "bun" +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/main.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/main.ts new file mode 100644 index 0000000000..4ada29ad22 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/main.ts @@ -0,0 +1,40 @@ +interface MetricCardData { + id: string + label: string + value: number + unit: 'currency' | 'count' | 'percent' + delta: number + hint: string +} + +// Returns the headline metric cards for the selected range and region. Values +// are mocked but internally consistent (revenue / orders ≈ avg order value). +const baseByRegion: Record = { + all: { revenue: 482_300, orders: 412, units: 1840, refunds: 11_900 }, + 'North America': { revenue: 211_400, orders: 168, units: 770, refunds: 4_200 }, + EMEA: { revenue: 142_900, orders: 121, units: 560, refunds: 3_500 }, + APAC: { revenue: 86_500, orders: 78, units: 340, refunds: 2_600 }, + LATAM: { revenue: 41_500, orders: 45, units: 170, refunds: 1_600 } +} + +export async function main({ + from, + to, + region +}: { + from: string + to: string + region: string +}): Promise<{ cards: MetricCardData[]; generatedAt: string }> { + const base = baseByRegion[region] ?? baseByRegion.all + const aov = base.orders === 0 ? 0 : Math.round(base.revenue / base.orders) + const cards: MetricCardData[] = [ + { id: 'revenue', label: 'Total Revenue', value: base.revenue, unit: 'currency', delta: 0.082, hint: `Booked revenue ${from} – ${to}` }, + { id: 'orders', label: 'Orders', value: base.orders, unit: 'count', delta: 0.041, hint: 'Revenue-bearing orders in range' }, + { id: 'aov', label: 'Avg Order Value', value: aov, unit: 'currency', delta: -0.013, hint: 'Total revenue / order count' }, + { id: 'units', label: 'Units Sold', value: base.units, unit: 'count', delta: 0.067, hint: 'Total units in range' }, + { id: 'refunds', label: 'Refunded', value: base.refunds, unit: 'currency', delta: -0.021, hint: 'Revenue lost to refunds' }, + { id: 'conversion', label: 'Conversion', value: 0.187, unit: 'percent', delta: 0.009, hint: 'Sessions that became orders' } + ] + return { cards, generatedAt: new Date().toISOString() } +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/meta.json b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/meta.json new file mode 100644 index 0000000000..1fbc3337c9 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadMetrics/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Load Metrics", + "language": "bun" +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/main.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/main.ts new file mode 100644 index 0000000000..27a5396f2e --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/main.ts @@ -0,0 +1,54 @@ +type OrderStatus = 'paid' | 'shipped' | 'delivered' | 'pending' | 'refunded' | 'cancelled' + +interface Order { + id: string + placedAt: string + customer: string + product: string + sku: string + region: string + channel: string + rep: string + quantity: number + unitPrice: number + status: OrderStatus +} + +// Mocked order book. In a real deployment this would query the orders table; +// here it returns a representative slice so the table renders in preview. +const orders: Order[] = [ + { id: 'ORD-10001', placedAt: '2024-05-02T09:14:00Z', customer: 'Contoso Ltd', product: 'Aurora Analytics Suite', sku: 'ANL-100', region: 'North America', channel: 'direct', rep: 'Dana Wills', quantity: 3, unitPrice: 1195, status: 'delivered' }, + { id: 'ORD-10002', placedAt: '2024-05-03T11:42:00Z', customer: 'Fabrikam Inc', product: 'Borealis CRM', sku: 'CRM-210', region: 'EMEA', channel: 'partner', rep: 'Lena Fischer', quantity: 5, unitPrice: 880, status: 'shipped' }, + { id: 'ORD-10003', placedAt: '2024-05-05T15:03:00Z', customer: 'Tailspin Toys', product: 'Cascade Data Pipeline', sku: 'PIPE-330', region: 'APAC', channel: 'self-serve', rep: 'Sora Tanaka', quantity: 2, unitPrice: 640, status: 'paid' }, + { id: 'ORD-10004', placedAt: '2024-05-07T08:21:00Z', customer: 'Proseware Inc', product: 'Delta Insights', sku: 'INS-440', region: 'LATAM', channel: 'marketplace', rep: 'Diego Marin', quantity: 7, unitPrice: 315, status: 'delivered' }, + { id: 'ORD-10005', placedAt: '2024-05-09T13:58:00Z', customer: 'Litware Inc', product: 'Echo Monitoring', sku: 'MON-550', region: 'North America', channel: 'direct', rep: 'Owen Pratt', quantity: 4, unitPrice: 150, status: 'refunded' }, + { id: 'ORD-10006', placedAt: '2024-05-12T10:30:00Z', customer: 'Fourth Coffee', product: 'Helix Identity', sku: 'IDN-880', region: 'EMEA', channel: 'partner', rep: 'Aisha Khan', quantity: 6, unitPrice: 220, status: 'shipped' }, + { id: 'ORD-10007', placedAt: '2024-05-15T17:11:00Z', customer: 'Coho Vineyard', product: 'Kelvin Forecasting', sku: 'FCT-202', region: 'APAC', channel: 'direct', rep: 'Priya Nair', quantity: 1, unitPrice: 980, status: 'pending' }, + { id: 'ORD-10008', placedAt: '2024-05-18T12:05:00Z', customer: 'Alpine Ski House', product: 'Nimbus Compute', sku: 'CMP-505', region: 'North America', channel: 'self-serve', rep: 'Hugo Bernard', quantity: 8, unitPrice: 1100, status: 'delivered' }, + { id: 'ORD-10009', placedAt: '2024-05-22T14:47:00Z', customer: 'Trey Research', product: 'Onyx Security', sku: 'SEC-606', region: 'EMEA', channel: 'direct', rep: 'Sven Olsen', quantity: 2, unitPrice: 860, status: 'cancelled' }, + { id: 'ORD-10010', placedAt: '2024-05-26T16:39:00Z', customer: 'Blue Yonder Airlines', product: 'Polaris Reporting', sku: 'RPT-707', region: 'LATAM', channel: 'partner', rep: 'Mateo Russo', quantity: 9, unitPrice: 290, status: 'paid' } +] + +export async function main({ + from, + to, + region, + status +}: { + from: string + to: string + region: string + status: string +}): Promise<{ orders: Order[]; total: number }> { + let filtered = orders.filter((order) => { + const day = order.placedAt.slice(0, 10) + return day >= from && day <= to + }) + if (region && region !== 'all') { + filtered = filtered.filter((order) => order.region === region) + } + if (status && status !== 'all') { + filtered = filtered.filter((order) => order.status === status) + } + return { orders: filtered, total: filtered.length } +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/meta.json b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/meta.json new file mode 100644 index 0000000000..098e077560 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/backend/loadOrders/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Load Orders", + "language": "bun" +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/DateRangePicker.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/DateRangePicker.tsx new file mode 100644 index 0000000000..f0a76a09f3 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/DateRangePicker.tsx @@ -0,0 +1,45 @@ +import React from 'react' +import type { DateRange } from '../lib/api' +import { rangeForPreset } from '../lib/api' +import { formatDateShort } from '../lib/format' + +interface DateRangePickerProps { + preset: string + range: DateRange + onPresetChange: (preset: string, range: DateRange) => void +} + +const PRESETS: { id: string; label: string }[] = [ + { id: '7d', label: 'Last 7 days' }, + { id: '14d', label: 'Last 14 days' }, + { id: '30d', label: 'Last 30 days' }, + { id: 'qtd', label: 'Quarter to date' } +] + +export const DateRangePicker: React.FC = ({ + preset, + range, + onPresetChange +}) => { + return ( +
+ + + {formatDateShort(range.from)} – {formatDateShort(range.to)} + +
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/EmptyState.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/EmptyState.tsx new file mode 100644 index 0000000000..f4dc840c60 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/EmptyState.tsx @@ -0,0 +1,28 @@ +import React from 'react' + +interface EmptyStateProps { + title: string + description?: string + icon?: string + action?: React.ReactNode +} + +export const EmptyState: React.FC = ({ + title, + description, + icon = '📊', + action +}) => { + return ( +
+
+ {icon} +
+

{title}

+ {description ? ( +

{description}

+ ) : null} + {action ?
{action}
: null} +
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/ExportButton.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/ExportButton.tsx new file mode 100644 index 0000000000..6fc9b5adde --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/ExportButton.tsx @@ -0,0 +1,51 @@ +import React, { useState } from 'react' +import { requestExport } from '../lib/api' +import type { DateRange } from '../lib/api' + +interface ExportButtonProps { + range: DateRange + region: string +} + +export const ExportButton: React.FC = ({ range, region }) => { + const [busy, setBusy] = useState(false) + const [error, setError] = useState(null) + + const handleExport = async (format: 'csv' | 'json') => { + setBusy(true) + setError(null) + try { + const result = await requestExport(range, region, format) + const anchor = document.createElement('a') + anchor.href = result.url + anchor.download = `revenue-report.${format}` + anchor.click() + } catch (err) { + setError(err instanceof Error ? err.message : 'Export failed') + } finally { + setBusy(false) + } + } + + return ( +
+ + + {error ? {error} : null} +
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/FilterBar.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/FilterBar.tsx new file mode 100644 index 0000000000..38de9d6a9b --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/FilterBar.tsx @@ -0,0 +1,59 @@ +import React from 'react' +import type { DateRange } from '../lib/api' +import type { OrderStatus } from '../data/seedData' +import { REGIONS, ORDER_STATUSES, STATUS_LABELS } from '../data/seedData' +import { DateRangePicker } from './DateRangePicker' +import { ExportButton } from './ExportButton' + +interface FilterBarProps { + region: string + status: string + preset: string + range: DateRange + onRegionChange: (region: string) => void + onStatusChange: (status: string) => void + onPresetChange: (preset: string, range: DateRange) => void +} + +export const FilterBar: React.FC = ({ + region, + status, + preset, + range, + onRegionChange, + onStatusChange, + onPresetChange +}) => { + return ( +
+
+ + + +
+ +
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricCard.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricCard.tsx new file mode 100644 index 0000000000..23dcd90123 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricCard.tsx @@ -0,0 +1,40 @@ +import React from 'react' +import type { MetricCardData } from '../data/seedData' +import { formatCurrency, formatNumber, formatPercent, formatSignedPercent } from '../lib/format' + +interface MetricCardProps { + metric: MetricCardData + loading?: boolean +} + +function renderValue(metric: MetricCardData): string { + switch (metric.unit) { + case 'currency': + return formatCurrency(metric.value) + case 'percent': + return formatPercent(metric.value) + case 'count': + default: + return formatNumber(metric.value) + } +} + +export const MetricCard: React.FC = ({ metric, loading }) => { + const positive = metric.delta >= 0 + return ( +
+
+ {metric.label} + + {formatSignedPercent(metric.delta)} + +
+
+ {loading ? … : renderValue(metric)} +
+

{metric.hint}

+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricGrid.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricGrid.tsx new file mode 100644 index 0000000000..c42a8aa733 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/MetricGrid.tsx @@ -0,0 +1,18 @@ +import React from 'react' +import type { MetricCardData } from '../data/seedData' +import { MetricCard } from './MetricCard' + +interface MetricGridProps { + metrics: MetricCardData[] + loading?: boolean +} + +export const MetricGrid: React.FC = ({ metrics, loading }) => { + return ( +
+ {metrics.map((metric) => ( + + ))} +
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/OrdersTable.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/OrdersTable.tsx new file mode 100644 index 0000000000..fa0411c371 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/OrdersTable.tsx @@ -0,0 +1,117 @@ +import React, { useMemo, useState } from 'react' +import type { Order } from '../data/seedData' +import { StatusBadge } from './StatusBadge' +import { EmptyState } from './EmptyState' +import { formatCurrencyPrecise, formatDate, formatNumber, truncate } from '../lib/format' + +interface OrdersTableProps { + orders: Order[] + loading?: boolean +} + +type SortKey = 'placedAt' | 'customer' | 'lineTotal' | 'quantity' +type SortDir = 'asc' | 'desc' + +// The per-row line total a customer was charged: unit price times quantity. +function lineTotal(order: Order): number { + return order.quantity * order.unitPrice +} + +export const OrdersTable: React.FC = ({ orders, loading }) => { + const [sortKey, setSortKey] = useState('placedAt') + const [sortDir, setSortDir] = useState('desc') + + const sorted = useMemo(() => { + const copy = [...orders] + copy.sort((a, b) => { + let comparison = 0 + switch (sortKey) { + case 'customer': + comparison = a.customer.localeCompare(b.customer) + break + case 'lineTotal': + comparison = lineTotal(a) - lineTotal(b) + break + case 'quantity': + comparison = a.quantity - b.quantity + break + case 'placedAt': + default: + comparison = a.placedAt.localeCompare(b.placedAt) + break + } + return sortDir === 'asc' ? comparison : -comparison + }) + return copy + }, [orders, sortKey, sortDir]) + + const toggleSort = (key: SortKey) => { + if (key === sortKey) { + setSortDir((dir) => (dir === 'asc' ? 'desc' : 'asc')) + } else { + setSortKey(key) + setSortDir('desc') + } + } + + if (!loading && orders.length === 0) { + return ( + + ) + } + + const arrow = (key: SortKey) => (key === sortKey ? (sortDir === 'asc' ? '▲' : '▼') : '') + + return ( +
+ + + + + + + + + + + + + + + {sorted.map((order) => ( + + + + + + + + + + + ))} + +
toggleSort('placedAt')}> + Date {arrow('placedAt')} + toggleSort('customer')}> + Customer {arrow('customer')} + ProductRegion toggleSort('quantity')}> + Qty {arrow('quantity')} + Unit Price toggleSort('lineTotal')}> + Line Total {arrow('lineTotal')} + Status
{formatDate(order.placedAt)} + {truncate(order.customer, 24)} + {order.product}{order.region}{formatNumber(order.quantity)} + {formatCurrencyPrecise(order.unitPrice)} + + {formatCurrencyPrecise(lineTotal(order))} + + +
+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RegionTable.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RegionTable.tsx new file mode 100644 index 0000000000..41e47430b9 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RegionTable.tsx @@ -0,0 +1,52 @@ +import React, { useMemo } from 'react' +import type { Order } from '../data/seedData' +import { breakdownByRegion } from '../lib/aggregations' +import { formatCurrency, formatNumber, formatPercent } from '../lib/format' +import { EmptyState } from './EmptyState' + +interface RegionTableProps { + orders: Order[] +} + +export const RegionTable: React.FC = ({ orders }) => { + const rows = useMemo(() => breakdownByRegion(orders), [orders]) + const total = useMemo(() => rows.reduce((acc, row) => acc + row.revenue, 0), [rows]) + + if (rows.length === 0) { + return ( + + ) + } + + return ( +
+

Revenue by Region

+ + + + + + + + + + + {rows.map((row) => ( + + + + + + + ))} + +
RegionOrdersRevenueShare
{row.region}{formatNumber(row.orders)}{formatCurrency(row.revenue)} + {formatPercent(total === 0 ? 0 : row.revenue / total)} +
+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RevenueChart.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RevenueChart.tsx new file mode 100644 index 0000000000..f16fe779fe --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/RevenueChart.tsx @@ -0,0 +1,49 @@ +import React, { useMemo } from 'react' +import type { Order } from '../data/seedData' +import { dailyRevenue } from '../lib/aggregations' +import { formatCompact, formatDateShort } from '../lib/format' +import { EmptyState } from './EmptyState' + +interface RevenueChartProps { + orders: Order[] +} + +// Lightweight inline bar chart for daily revenue. Avoids a charting dependency +// by sizing flexed columns relative to the busiest day in the window. +export const RevenueChart: React.FC = ({ orders }) => { + const points = useMemo(() => dailyRevenue(orders), [orders]) + const max = useMemo(() => points.reduce((acc, point) => Math.max(acc, point.revenue), 0), [points]) + + if (points.length === 0) { + return ( + + ) + } + + return ( +
+

Daily Revenue

+
+ {points.map((point) => { + const heightPct = max === 0 ? 0 : Math.round((point.revenue / max) * 100) + return ( +
+
+ + {formatDateShort(point.date)} + +
+ ) + })} +
+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/Sidebar.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/Sidebar.tsx new file mode 100644 index 0000000000..5e92f4719b --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/Sidebar.tsx @@ -0,0 +1,50 @@ +import React from 'react' + +export type DashboardView = 'overview' | 'orders' | 'regions' | 'products' + +interface SidebarProps { + active: DashboardView + onSelect: (view: DashboardView) => void +} + +const NAV_ITEMS: { id: DashboardView; label: string; icon: string }[] = [ + { id: 'overview', label: 'Overview', icon: '📈' }, + { id: 'orders', label: 'Orders', icon: '🧾' }, + { id: 'regions', label: 'Regions', icon: '🌍' }, + { id: 'products', label: 'Products', icon: '📦' } +] + +export const Sidebar: React.FC = ({ active, onSelect }) => { + return ( + + ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/StatusBadge.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/StatusBadge.tsx new file mode 100644 index 0000000000..114d2a7073 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/StatusBadge.tsx @@ -0,0 +1,26 @@ +import React from 'react' +import type { OrderStatus } from '../data/seedData' +import { STATUS_LABELS } from '../data/seedData' + +interface StatusBadgeProps { + status: OrderStatus +} + +const STATUS_STYLES: Record = { + paid: 'bg-blue-100 text-blue-700', + shipped: 'bg-indigo-100 text-indigo-700', + delivered: 'bg-emerald-100 text-emerald-700', + pending: 'bg-amber-100 text-amber-700', + refunded: 'bg-rose-100 text-rose-700', + cancelled: 'bg-gray-200 text-gray-600' +} + +export const StatusBadge: React.FC = ({ status }) => { + return ( + + {STATUS_LABELS[status]} + + ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/SummaryPanel.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/SummaryPanel.tsx new file mode 100644 index 0000000000..0ec6a345bc --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/SummaryPanel.tsx @@ -0,0 +1,51 @@ +import React, { useMemo } from 'react' +import type { Order } from '../data/seedData' +import { summarizeRevenue } from '../lib/aggregations' +import { formatCurrency, formatCurrencyPrecise, formatNumber } from '../lib/format' + +interface SummaryPanelProps { + orders: Order[] + loading?: boolean +} + +// Headline revenue panel. It re-aggregates the orders client-side via +// summarizeRevenue so the totals stay in sync with whatever filter the user +// has applied, without waiting for another backend round trip. +export const SummaryPanel: React.FC = ({ orders, loading }) => { + const summary = useMemo(() => summarizeRevenue(orders), [orders]) + + const tiles = [ + { label: 'Total Revenue', value: formatCurrency(summary.totalRevenue), emphasis: true }, + { label: 'Net Revenue', value: formatCurrency(summary.netRevenue) }, + { label: 'Orders', value: formatNumber(summary.totalOrders) }, + { label: 'Avg Order Value', value: formatCurrencyPrecise(summary.averageOrderValue) }, + { label: 'Units Sold', value: formatNumber(summary.unitsSold) }, + { label: 'Refunded', value: formatCurrency(summary.refundedRevenue) } + ] + + return ( +
+
+

Revenue Summary

+ {loading ? Refreshing… : null} +
+
+ {tiles.map((tile) => ( +
+
+ {tile.label} +
+
+ {tile.value} +
+
+ ))} +
+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/TopProducts.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/TopProducts.tsx new file mode 100644 index 0000000000..9876e9701d --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/components/TopProducts.tsx @@ -0,0 +1,55 @@ +import React, { useMemo } from 'react' +import type { Order } from '../data/seedData' +import { topProducts } from '../lib/aggregations' +import { formatCurrency } from '../lib/format' +import { EmptyState } from './EmptyState' + +interface TopProductsProps { + orders: Order[] + limit?: number +} + +export const TopProducts: React.FC = ({ orders, limit = 5 }) => { + const products = useMemo(() => topProducts(orders, limit), [orders, limit]) + const max = useMemo( + () => products.reduce((acc, item) => Math.max(acc, item.revenue), 0), + [products] + ) + + if (products.length === 0) { + return ( + + ) + } + + return ( +
+

Top Products

+
    + {products.map((item, index) => { + const widthPct = max === 0 ? 0 : Math.round((item.revenue / max) * 100) + return ( +
  • +
    + + {index + 1}. {item.product} + + {formatCurrency(item.revenue)} +
    +
    +
    +
    +
    • + ) + })} +
+
+ ) +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/data/seedData.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/data/seedData.ts new file mode 100644 index 0000000000..742a83be3d --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/data/seedData.ts @@ -0,0 +1,5051 @@ +// Synthetic seed data for the analytics dashboard. +// +// This module is the app's offline data source: the backend runnables filter +// and aggregate these rows, and a few components fall back to them when a +// backend call is unavailable in preview. The array is intentionally large and +// varied so the dashboard renders realistic totals, regional splits, and time +// series. It is plain committed source — do not regenerate it at load time. + +export type OrderStatus = + | 'paid' + | 'shipped' + | 'delivered' + | 'pending' + | 'refunded' + | 'cancelled' + +export interface Order { + id: string + placedAt: string + customer: string + product: string + sku: string + region: string + channel: string + rep: string + quantity: number + unitPrice: number + status: OrderStatus +} + +export interface MetricCardData { + id: string + label: string + value: number + unit: 'currency' | 'count' | 'percent' + delta: number + hint: string +} + +export const PRODUCT_CATALOG: { name: string; sku: string; listPrice: number }[] = [ + { name: "Aurora Analytics Suite", sku: "ANL-100", listPrice: 1200 }, + { name: "Borealis CRM", sku: "CRM-210", listPrice: 890 }, + { name: "Cascade Data Pipeline", sku: "PIPE-330", listPrice: 640 }, + { name: "Delta Insights", sku: "INS-440", listPrice: 320 }, + { name: "Echo Monitoring", sku: "MON-550", listPrice: 150 }, + { name: "Fjord Storage", sku: "STO-660", listPrice: 75 }, + { name: "Glacier Backup", sku: "BAK-770", listPrice: 45 }, + { name: "Helix Identity", sku: "IDN-880", listPrice: 220 }, + { name: "Ion Messaging", sku: "MSG-990", listPrice: 60 }, + { name: "Juniper Workflow", sku: "WFL-101", listPrice: 410 }, + { name: "Kelvin Forecasting", sku: "FCT-202", listPrice: 980 }, + { name: "Lumen Dashboards", sku: "DSH-303", listPrice: 520 }, + { name: "Meridian ETL", sku: "ETL-404", listPrice: 730 }, + { name: "Nimbus Compute", sku: "CMP-505", listPrice: 1100 }, + { name: "Onyx Security", sku: "SEC-606", listPrice: 860 }, + { name: "Polaris Reporting", sku: "RPT-707", listPrice: 290 }, +] + +export const REGIONS: string[] = [ + "North America", + "EMEA", + "APAC", + "LATAM", +] + +export const ORDER_STATUSES: OrderStatus[] = [ + "paid", + "shipped", + "delivered", + "pending", + "refunded", + "cancelled", +] + +export const STATUS_LABELS: Record = { + paid: 'Paid', + shipped: 'Shipped', + delivered: 'Delivered', + pending: 'Pending', + refunded: 'Refunded', + cancelled: 'Cancelled' +} + +export const seedOrders: Order[] = [ + { + id: "ORD-10070", + placedAt: "2024-05-01T02:15:00Z", + customer: "Wingtip Toys", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 71, + status: "pending" + }, + { + id: "ORD-10245", + placedAt: "2024-05-01T03:12:00Z", + customer: "Adventure Works", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "partner", + rep: "Priya Nair", + quantity: 8, + unitPrice: 161, + status: "shipped" + }, + { + id: "ORD-10368", + placedAt: "2024-05-01T06:05:00Z", + customer: "Blue Yonder Airlines", + product: "Polaris Reporting", + sku: "RPT-707", + region: "EMEA", + channel: "direct", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 295, + status: "paid" + }, + { + id: "ORD-10235", + placedAt: "2024-05-01T07:51:00Z", + customer: "Contoso Ltd", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 7, + unitPrice: 997, + status: "shipped" + }, + { + id: "ORD-10142", + placedAt: "2024-05-01T08:59:00Z", + customer: "Lucerne Publishing", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "marketplace", + rep: "Priya Nair", + quantity: 6, + unitPrice: 1094, + status: "refunded" + }, + { + id: "ORD-10300", + placedAt: "2024-05-01T12:08:00Z", + customer: "Wide World Importers", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "EMEA", + channel: "self-serve", + rep: "Priya Nair", + quantity: 7, + unitPrice: 516, + status: "shipped" + }, + { + id: "ORD-10229", + placedAt: "2024-05-01T12:26:00Z", + customer: "Lucerne Publishing", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "marketplace", + rep: "Sven Olsen", + quantity: 4, + unitPrice: 147, + status: "refunded" + }, + { + id: "ORD-10091", + placedAt: "2024-05-01T14:29:00Z", + customer: "Wingtip Toys", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "direct", + rep: "Diego Marin", + quantity: 2, + unitPrice: 984, + status: "delivered" + }, + { + id: "ORD-10329", + placedAt: "2024-05-01T19:07:00Z", + customer: "City Power & Light", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 5, + unitPrice: 59, + status: "delivered" + }, + { + id: "ORD-10200", + placedAt: "2024-05-01T20:25:00Z", + customer: "Fourth Coffee", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 223, + status: "delivered" + }, + { + id: "ORD-10177", + placedAt: "2024-05-01T21:13:00Z", + customer: "Wingtip Toys", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 1188, + status: "paid" + }, + { + id: "ORD-10101", + placedAt: "2024-05-01T21:21:00Z", + customer: "Fabrikam Inc", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 1, + unitPrice: 155, + status: "shipped" + }, + { + id: "ORD-10148", + placedAt: "2024-05-01T21:24:00Z", + customer: "Coho Vineyard", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "partner", + rep: "Sven Olsen", + quantity: 7, + unitPrice: 313, + status: "refunded" + }, + { + id: "ORD-10358", + placedAt: "2024-05-02T00:56:00Z", + customer: "Blue Yonder Airlines", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 63, + status: "cancelled" + }, + { + id: "ORD-10059", + placedAt: "2024-05-02T06:34:00Z", + customer: "Coho Vineyard", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "partner", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 990, + status: "refunded" + }, + { + id: "ORD-10279", + placedAt: "2024-05-02T13:07:00Z", + customer: "Wide World Importers", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "direct", + rep: "Priya Nair", + quantity: 7, + unitPrice: 47, + status: "delivered" + }, + { + id: "ORD-10215", + placedAt: "2024-05-02T14:05:00Z", + customer: "Adventure Works", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "direct", + rep: "Sora Tanaka", + quantity: 1, + unitPrice: 29, + status: "pending" + }, + { + id: "ORD-10341", + placedAt: "2024-05-02T15:19:00Z", + customer: "Northwind Traders", + product: "Echo Monitoring", + sku: "MON-550", + region: "North America", + channel: "direct", + rep: "Sora Tanaka", + quantity: 5, + unitPrice: 160, + status: "paid" + }, + { + id: "ORD-10039", + placedAt: "2024-05-02T15:24:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 4, + unitPrice: 46, + status: "paid" + }, + { + id: "ORD-10380", + placedAt: "2024-05-02T18:33:00Z", + customer: "Margies Travel", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 5, + unitPrice: 537, + status: "shipped" + }, + { + id: "ORD-10083", + placedAt: "2024-05-02T21:21:00Z", + customer: "Fabrikam Inc", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "APAC", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 634, + status: "shipped" + }, + { + id: "ORD-10233", + placedAt: "2024-05-02T22:38:00Z", + customer: "Contoso Ltd", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 71, + status: "delivered" + }, + { + id: "ORD-10243", + placedAt: "2024-05-03T03:42:00Z", + customer: "Northwind Traders", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "self-serve", + rep: "Diego Marin", + quantity: 2, + unitPrice: 634, + status: "paid" + }, + { + id: "ORD-10269", + placedAt: "2024-05-03T03:43:00Z", + customer: "City Power & Light", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 745, + status: "shipped" + }, + { + id: "ORD-10230", + placedAt: "2024-05-03T07:54:00Z", + customer: "Adventure Works", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 73, + status: "shipped" + }, + { + id: "ORD-10354", + placedAt: "2024-05-03T10:30:00Z", + customer: "Humongous Insurance", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "partner", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 879, + status: "refunded" + }, + { + id: "ORD-10256", + placedAt: "2024-05-03T12:01:00Z", + customer: "Tailspin Toys", + product: "Polaris Reporting", + sku: "RPT-707", + region: "LATAM", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 298, + status: "delivered" + }, + { + id: "ORD-10146", + placedAt: "2024-05-03T14:03:00Z", + customer: "Fourth Coffee", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "marketplace", + rep: "Sven Olsen", + quantity: 8, + unitPrice: 895, + status: "cancelled" + }, + { + id: "ORD-10162", + placedAt: "2024-05-03T14:09:00Z", + customer: "Margies Travel", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 2, + unitPrice: 887, + status: "shipped" + }, + { + id: "ORD-10313", + placedAt: "2024-05-03T21:27:00Z", + customer: "Fourth Coffee", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 46, + status: "refunded" + }, + { + id: "ORD-10282", + placedAt: "2024-05-03T22:46:00Z", + customer: "Lucerne Publishing", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "self-serve", + rep: "Diego Marin", + quantity: 7, + unitPrice: 405, + status: "delivered" + }, + { + id: "ORD-10224", + placedAt: "2024-05-03T23:05:00Z", + customer: "Contoso Ltd", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "self-serve", + rep: "Diego Marin", + quantity: 8, + unitPrice: 308, + status: "shipped" + }, + { + id: "ORD-10372", + placedAt: "2024-05-03T23:35:00Z", + customer: "Lucerne Publishing", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "marketplace", + rep: "Priya Nair", + quantity: 2, + unitPrice: 314, + status: "delivered" + }, + { + id: "ORD-10073", + placedAt: "2024-05-04T02:12:00Z", + customer: "City Power & Light", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 55, + status: "delivered" + }, + { + id: "ORD-10323", + placedAt: "2024-05-04T06:09:00Z", + customer: "Litware Inc", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "direct", + rep: "Priya Nair", + quantity: 1, + unitPrice: 635, + status: "shipped" + }, + { + id: "ORD-10012", + placedAt: "2024-05-04T06:18:00Z", + customer: "Proseware Inc", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "North America", + channel: "marketplace", + rep: "Dana Wills", + quantity: 4, + unitPrice: 526, + status: "shipped" + }, + { + id: "ORD-10124", + placedAt: "2024-05-04T06:28:00Z", + customer: "Litware Inc", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 2, + unitPrice: 503, + status: "refunded" + }, + { + id: "ORD-10338", + placedAt: "2024-05-04T06:32:00Z", + customer: "Northwind Traders", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "direct", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 895, + status: "cancelled" + }, + { + id: "ORD-10194", + placedAt: "2024-05-04T07:15:00Z", + customer: "Northwind Traders", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "direct", + rep: "Dana Wills", + quantity: 6, + unitPrice: 881, + status: "delivered" + }, + { + id: "ORD-10006", + placedAt: "2024-05-04T11:55:00Z", + customer: "Litware Inc", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "partner", + rep: "Sven Olsen", + quantity: 6, + unitPrice: 85, + status: "delivered" + }, + { + id: "ORD-10183", + placedAt: "2024-05-04T14:27:00Z", + customer: "Humongous Insurance", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "partner", + rep: "Diego Marin", + quantity: 1, + unitPrice: 62, + status: "paid" + }, + { + id: "ORD-10079", + placedAt: "2024-05-04T19:21:00Z", + customer: "Litware Inc", + product: "Onyx Security", + sku: "SEC-606", + region: "EMEA", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 840, + status: "delivered" + }, + { + id: "ORD-10304", + placedAt: "2024-05-04T20:22:00Z", + customer: "Graphic Design Institute", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "direct", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 287, + status: "shipped" + }, + { + id: "ORD-10020", + placedAt: "2024-05-04T20:56:00Z", + customer: "School of Fine Art", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "direct", + rep: "Priya Nair", + quantity: 3, + unitPrice: 324, + status: "delivered" + }, + { + id: "ORD-10133", + placedAt: "2024-05-05T02:28:00Z", + customer: "Coho Vineyard", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "partner", + rep: "Dana Wills", + quantity: 1, + unitPrice: 152, + status: "refunded" + }, + { + id: "ORD-10028", + placedAt: "2024-05-05T02:50:00Z", + customer: "Northwind Traders", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "marketplace", + rep: "Dana Wills", + quantity: 7, + unitPrice: 513, + status: "delivered" + }, + { + id: "ORD-10097", + placedAt: "2024-05-05T08:20:00Z", + customer: "Lucerne Publishing", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "partner", + rep: "Dana Wills", + quantity: 7, + unitPrice: 1218, + status: "delivered" + }, + { + id: "ORD-10117", + placedAt: "2024-05-05T08:35:00Z", + customer: "Northwind Traders", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "marketplace", + rep: "Diego Marin", + quantity: 5, + unitPrice: 160, + status: "paid" + }, + { + id: "ORD-10109", + placedAt: "2024-05-05T09:18:00Z", + customer: "Fourth Coffee", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 4, + unitPrice: 734, + status: "delivered" + }, + { + id: "ORD-10285", + placedAt: "2024-05-05T11:54:00Z", + customer: "Alpine Ski House", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 711, + status: "shipped" + }, + { + id: "ORD-10107", + placedAt: "2024-05-05T12:04:00Z", + customer: "Margies Travel", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "partner", + rep: "Diego Marin", + quantity: 1, + unitPrice: 986, + status: "cancelled" + }, + { + id: "ORD-10366", + placedAt: "2024-05-05T17:27:00Z", + customer: "Blue Yonder Airlines", + product: "Nimbus Compute", + sku: "CMP-505", + region: "APAC", + channel: "partner", + rep: "Sven Olsen", + quantity: 6, + unitPrice: 1108, + status: "shipped" + }, + { + id: "ORD-10356", + placedAt: "2024-05-05T19:37:00Z", + customer: "Lucerne Publishing", + product: "Delta Insights", + sku: "INS-440", + region: "LATAM", + channel: "marketplace", + rep: "Dana Wills", + quantity: 2, + unitPrice: 328, + status: "refunded" + }, + { + id: "ORD-10052", + placedAt: "2024-05-05T20:26:00Z", + customer: "Adventure Works", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "direct", + rep: "Mateo Russo", + quantity: 2, + unitPrice: 312, + status: "delivered" + }, + { + id: "ORD-10367", + placedAt: "2024-05-05T23:03:00Z", + customer: "Litware Inc", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 5, + unitPrice: 879, + status: "pending" + }, + { + id: "ORD-10234", + placedAt: "2024-05-05T23:09:00Z", + customer: "Margies Travel", + product: "Juniper Workflow", + sku: "WFL-101", + region: "EMEA", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 415, + status: "delivered" + }, + { + id: "ORD-10280", + placedAt: "2024-05-06T03:20:00Z", + customer: "Wide World Importers", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "partner", + rep: "Priya Nair", + quantity: 6, + unitPrice: 205, + status: "shipped" + }, + { + id: "ORD-10221", + placedAt: "2024-05-06T07:55:00Z", + customer: "Litware Inc", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "partner", + rep: "Owen Pratt", + quantity: 1, + unitPrice: 718, + status: "refunded" + }, + { + id: "ORD-10332", + placedAt: "2024-05-06T08:38:00Z", + customer: "Lucerne Publishing", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 1, + unitPrice: 504, + status: "delivered" + }, + { + id: "ORD-10121", + placedAt: "2024-05-06T11:55:00Z", + customer: "Wide World Importers", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "marketplace", + rep: "Dana Wills", + quantity: 5, + unitPrice: 50, + status: "delivered" + }, + { + id: "ORD-10128", + placedAt: "2024-05-06T13:54:00Z", + customer: "City Power & Light", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "marketplace", + rep: "Mateo Russo", + quantity: 7, + unitPrice: 299, + status: "delivered" + }, + { + id: "ORD-10238", + placedAt: "2024-05-06T15:53:00Z", + customer: "Wide World Importers", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "partner", + rep: "Sora Tanaka", + quantity: 2, + unitPrice: 1090, + status: "shipped" + }, + { + id: "ORD-10193", + placedAt: "2024-05-06T15:55:00Z", + customer: "Trey Research", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "EMEA", + channel: "marketplace", + rep: "Mateo Russo", + quantity: 6, + unitPrice: 1198, + status: "delivered" + }, + { + id: "ORD-10111", + placedAt: "2024-05-06T16:37:00Z", + customer: "Fourth Coffee", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 868, + status: "delivered" + }, + { + id: "ORD-10220", + placedAt: "2024-05-06T16:57:00Z", + customer: "Margies Travel", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 6, + unitPrice: 525, + status: "pending" + }, + { + id: "ORD-10076", + placedAt: "2024-05-06T17:25:00Z", + customer: "Tailspin Toys", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "EMEA", + channel: "partner", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 500, + status: "refunded" + }, + { + id: "ORD-10369", + placedAt: "2024-05-06T17:43:00Z", + customer: "Coho Vineyard", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "LATAM", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 1190, + status: "shipped" + }, + { + id: "ORD-10378", + placedAt: "2024-05-06T19:28:00Z", + customer: "Fourth Coffee", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 7, + unitPrice: 397, + status: "delivered" + }, + { + id: "ORD-10283", + placedAt: "2024-05-06T20:02:00Z", + customer: "Margies Travel", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "APAC", + channel: "self-serve", + rep: "Dana Wills", + quantity: 4, + unitPrice: 983, + status: "shipped" + }, + { + id: "ORD-10277", + placedAt: "2024-05-06T20:35:00Z", + customer: "Blue Yonder Airlines", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 6, + unitPrice: 149, + status: "delivered" + }, + { + id: "ORD-10212", + placedAt: "2024-05-07T02:01:00Z", + customer: "Wide World Importers", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 4, + unitPrice: 337, + status: "pending" + }, + { + id: "ORD-10340", + placedAt: "2024-05-07T03:23:00Z", + customer: "Blue Yonder Airlines", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "marketplace", + rep: "Dana Wills", + quantity: 7, + unitPrice: 332, + status: "paid" + }, + { + id: "ORD-10044", + placedAt: "2024-05-07T04:10:00Z", + customer: "Proseware Inc", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "direct", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 526, + status: "delivered" + }, + { + id: "ORD-10293", + placedAt: "2024-05-07T05:52:00Z", + customer: "Fourth Coffee", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 166, + status: "shipped" + }, + { + id: "ORD-10058", + placedAt: "2024-05-07T06:38:00Z", + customer: "Litware Inc", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "direct", + rep: "Aisha Khan", + quantity: 2, + unitPrice: 423, + status: "delivered" + }, + { + id: "ORD-10298", + placedAt: "2024-05-07T06:47:00Z", + customer: "Wide World Importers", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "partner", + rep: "Owen Pratt", + quantity: 1, + unitPrice: 413, + status: "shipped" + }, + { + id: "ORD-10259", + placedAt: "2024-05-07T11:51:00Z", + customer: "Trey Research", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "direct", + rep: "Dana Wills", + quantity: 5, + unitPrice: 627, + status: "cancelled" + }, + { + id: "ORD-10295", + placedAt: "2024-05-07T14:13:00Z", + customer: "Humongous Insurance", + product: "Glacier Backup", + sku: "BAK-770", + region: "LATAM", + channel: "marketplace", + rep: "Dana Wills", + quantity: 8, + unitPrice: 60, + status: "delivered" + }, + { + id: "ORD-10219", + placedAt: "2024-05-07T14:15:00Z", + customer: "Northwind Traders", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 4, + unitPrice: 983, + status: "refunded" + }, + { + id: "ORD-10043", + placedAt: "2024-05-07T15:24:00Z", + customer: "Humongous Insurance", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "APAC", + channel: "direct", + rep: "Lena Fischer", + quantity: 1, + unitPrice: 978, + status: "delivered" + }, + { + id: "ORD-10271", + placedAt: "2024-05-07T15:28:00Z", + customer: "Margies Travel", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 849, + status: "delivered" + }, + { + id: "ORD-10050", + placedAt: "2024-05-07T19:40:00Z", + customer: "Wide World Importers", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 5, + unitPrice: 888, + status: "delivered" + }, + { + id: "ORD-10167", + placedAt: "2024-05-07T21:03:00Z", + customer: "Humongous Insurance", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "partner", + rep: "Aisha Khan", + quantity: 8, + unitPrice: 49, + status: "pending" + }, + { + id: "ORD-10328", + placedAt: "2024-05-08T00:23:00Z", + customer: "Wingtip Toys", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "direct", + rep: "Mateo Russo", + quantity: 7, + unitPrice: 204, + status: "shipped" + }, + { + id: "ORD-10089", + placedAt: "2024-05-08T01:57:00Z", + customer: "Adventure Works", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "direct", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 71, + status: "delivered" + }, + { + id: "ORD-10166", + placedAt: "2024-05-08T02:21:00Z", + customer: "Margies Travel", + product: "Fjord Storage", + sku: "STO-660", + region: "LATAM", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 5, + unitPrice: 86, + status: "cancelled" + }, + { + id: "ORD-10067", + placedAt: "2024-05-08T03:31:00Z", + customer: "Fourth Coffee", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "LATAM", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 645, + status: "delivered" + }, + { + id: "ORD-10004", + placedAt: "2024-05-08T08:06:00Z", + customer: "Fourth Coffee", + product: "Delta Insights", + sku: "INS-440", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 8, + unitPrice: 326, + status: "paid" + }, + { + id: "ORD-10174", + placedAt: "2024-05-08T16:11:00Z", + customer: "Adventure Works", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 1113, + status: "refunded" + }, + { + id: "ORD-10204", + placedAt: "2024-05-08T18:09:00Z", + customer: "Contoso Ltd", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "North America", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 1, + unitPrice: 510, + status: "delivered" + }, + { + id: "ORD-10173", + placedAt: "2024-05-08T20:58:00Z", + customer: "School of Fine Art", + product: "Meridian ETL", + sku: "ETL-404", + region: "North America", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 6, + unitPrice: 715, + status: "paid" + }, + { + id: "ORD-10297", + placedAt: "2024-05-08T22:20:00Z", + customer: "Tailspin Toys", + product: "Ion Messaging", + sku: "MSG-990", + region: "APAC", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 5, + unitPrice: 56, + status: "delivered" + }, + { + id: "ORD-10144", + placedAt: "2024-05-08T22:40:00Z", + customer: "Lucerne Publishing", + product: "Polaris Reporting", + sku: "RPT-707", + region: "EMEA", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 6, + unitPrice: 287, + status: "paid" + }, + { + id: "ORD-10123", + placedAt: "2024-05-09T00:54:00Z", + customer: "Tailspin Toys", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 8, + unitPrice: 966, + status: "cancelled" + }, + { + id: "ORD-10038", + placedAt: "2024-05-09T02:54:00Z", + customer: "Fourth Coffee", + product: "Fjord Storage", + sku: "STO-660", + region: "LATAM", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 92, + status: "paid" + }, + { + id: "ORD-10330", + placedAt: "2024-05-09T04:15:00Z", + customer: "Margies Travel", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "partner", + rep: "Owen Pratt", + quantity: 1, + unitPrice: 417, + status: "paid" + }, + { + id: "ORD-10098", + placedAt: "2024-05-09T05:21:00Z", + customer: "Humongous Insurance", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "partner", + rep: "Aisha Khan", + quantity: 6, + unitPrice: 907, + status: "delivered" + }, + { + id: "ORD-10377", + placedAt: "2024-05-09T06:21:00Z", + customer: "City Power & Light", + product: "Ion Messaging", + sku: "MSG-990", + region: "APAC", + channel: "partner", + rep: "Sora Tanaka", + quantity: 7, + unitPrice: 77, + status: "paid" + }, + { + id: "ORD-10239", + placedAt: "2024-05-09T07:32:00Z", + customer: "Tailspin Toys", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "marketplace", + rep: "Diego Marin", + quantity: 1, + unitPrice: 848, + status: "paid" + }, + { + id: "ORD-10337", + placedAt: "2024-05-09T14:16:00Z", + customer: "City Power & Light", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 6, + unitPrice: 1191, + status: "shipped" + }, + { + id: "ORD-10251", + placedAt: "2024-05-09T15:22:00Z", + customer: "Lucerne Publishing", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "marketplace", + rep: "Priya Nair", + quantity: 5, + unitPrice: 976, + status: "shipped" + }, + { + id: "ORD-10305", + placedAt: "2024-05-09T19:38:00Z", + customer: "Margies Travel", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "direct", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 1191, + status: "shipped" + }, + { + id: "ORD-10240", + placedAt: "2024-05-09T20:34:00Z", + customer: "Contoso Ltd", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 4, + unitPrice: 301, + status: "shipped" + }, + { + id: "ORD-10090", + placedAt: "2024-05-09T23:38:00Z", + customer: "Contoso Ltd", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 424, + status: "pending" + }, + { + id: "ORD-10060", + placedAt: "2024-05-10T01:13:00Z", + customer: "Alpine Ski House", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "direct", + rep: "Dana Wills", + quantity: 8, + unitPrice: 532, + status: "delivered" + }, + { + id: "ORD-10203", + placedAt: "2024-05-10T02:13:00Z", + customer: "Northwind Traders", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "APAC", + channel: "direct", + rep: "Dana Wills", + quantity: 6, + unitPrice: 985, + status: "shipped" + }, + { + id: "ORD-10056", + placedAt: "2024-05-10T02:17:00Z", + customer: "Wingtip Toys", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 222, + status: "shipped" + }, + { + id: "ORD-10362", + placedAt: "2024-05-10T02:34:00Z", + customer: "Humongous Insurance", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "direct", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 413, + status: "shipped" + }, + { + id: "ORD-10241", + placedAt: "2024-05-10T04:29:00Z", + customer: "Margies Travel", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 1189, + status: "pending" + }, + { + id: "ORD-10344", + placedAt: "2024-05-10T09:07:00Z", + customer: "Alpine Ski House", + product: "Helix Identity", + sku: "IDN-880", + region: "LATAM", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 208, + status: "delivered" + }, + { + id: "ORD-10066", + placedAt: "2024-05-10T09:13:00Z", + customer: "School of Fine Art", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 909, + status: "shipped" + }, + { + id: "ORD-10084", + placedAt: "2024-05-10T10:14:00Z", + customer: "Coho Vineyard", + product: "Delta Insights", + sku: "INS-440", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 325, + status: "delivered" + }, + { + id: "ORD-10349", + placedAt: "2024-05-10T10:19:00Z", + customer: "Fourth Coffee", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "partner", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 721, + status: "paid" + }, + { + id: "ORD-10262", + placedAt: "2024-05-10T13:05:00Z", + customer: "Fabrikam Inc", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "partner", + rep: "Sora Tanaka", + quantity: 6, + unitPrice: 90, + status: "shipped" + }, + { + id: "ORD-10376", + placedAt: "2024-05-10T14:01:00Z", + customer: "Northwind Traders", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 5, + unitPrice: 226, + status: "delivered" + }, + { + id: "ORD-10278", + placedAt: "2024-05-10T15:14:00Z", + customer: "Proseware Inc", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 5, + unitPrice: 76, + status: "delivered" + }, + { + id: "ORD-10211", + placedAt: "2024-05-10T15:38:00Z", + customer: "Adventure Works", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "partner", + rep: "Sven Olsen", + quantity: 2, + unitPrice: 622, + status: "paid" + }, + { + id: "ORD-10151", + placedAt: "2024-05-10T16:46:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "LATAM", + channel: "direct", + rep: "Priya Nair", + quantity: 5, + unitPrice: 56, + status: "delivered" + }, + { + id: "ORD-10063", + placedAt: "2024-05-10T19:38:00Z", + customer: "Adventure Works", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 872, + status: "shipped" + }, + { + id: "ORD-10347", + placedAt: "2024-05-10T21:40:00Z", + customer: "Adventure Works", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 8, + unitPrice: 983, + status: "refunded" + }, + { + id: "ORD-10034", + placedAt: "2024-05-10T22:55:00Z", + customer: "City Power & Light", + product: "Borealis CRM", + sku: "CRM-210", + region: "North America", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 880, + status: "delivered" + }, + { + id: "ORD-10131", + placedAt: "2024-05-11T02:18:00Z", + customer: "Blue Yonder Airlines", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "direct", + rep: "Diego Marin", + quantity: 2, + unitPrice: 642, + status: "shipped" + }, + { + id: "ORD-10218", + placedAt: "2024-05-11T02:18:00Z", + customer: "Fourth Coffee", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "direct", + rep: "Priya Nair", + quantity: 1, + unitPrice: 393, + status: "delivered" + }, + { + id: "ORD-10361", + placedAt: "2024-05-11T02:44:00Z", + customer: "Northwind Traders", + product: "Ion Messaging", + sku: "MSG-990", + region: "APAC", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 1, + unitPrice: 43, + status: "pending" + }, + { + id: "ORD-10199", + placedAt: "2024-05-11T04:13:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 39, + status: "shipped" + }, + { + id: "ORD-10127", + placedAt: "2024-05-11T08:16:00Z", + customer: "Adventure Works", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "marketplace", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 877, + status: "refunded" + }, + { + id: "ORD-10355", + placedAt: "2024-05-11T10:52:00Z", + customer: "Humongous Insurance", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "direct", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 631, + status: "delivered" + }, + { + id: "ORD-10320", + placedAt: "2024-05-11T13:30:00Z", + customer: "Northwind Traders", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 3, + unitPrice: 286, + status: "delivered" + }, + { + id: "ORD-10273", + placedAt: "2024-05-11T16:17:00Z", + customer: "Fabrikam Inc", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "self-serve", + rep: "Priya Nair", + quantity: 5, + unitPrice: 1209, + status: "shipped" + }, + { + id: "ORD-10263", + placedAt: "2024-05-11T17:44:00Z", + customer: "Fabrikam Inc", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "marketplace", + rep: "Priya Nair", + quantity: 6, + unitPrice: 30, + status: "refunded" + }, + { + id: "ORD-10172", + placedAt: "2024-05-11T18:43:00Z", + customer: "Adventure Works", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "marketplace", + rep: "Diego Marin", + quantity: 7, + unitPrice: 518, + status: "shipped" + }, + { + id: "ORD-10255", + placedAt: "2024-05-11T21:49:00Z", + customer: "City Power & Light", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "direct", + rep: "Dana Wills", + quantity: 8, + unitPrice: 858, + status: "shipped" + }, + { + id: "ORD-10275", + placedAt: "2024-05-11T22:08:00Z", + customer: "Adventure Works", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "direct", + rep: "Hugo Bernard", + quantity: 8, + unitPrice: 631, + status: "delivered" + }, + { + id: "ORD-10048", + placedAt: "2024-05-12T08:34:00Z", + customer: "Trey Research", + product: "Polaris Reporting", + sku: "RPT-707", + region: "LATAM", + channel: "partner", + rep: "Sora Tanaka", + quantity: 8, + unitPrice: 276, + status: "paid" + }, + { + id: "ORD-10141", + placedAt: "2024-05-12T12:59:00Z", + customer: "Graphic Design Institute", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "direct", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 710, + status: "shipped" + }, + { + id: "ORD-10319", + placedAt: "2024-05-12T14:48:00Z", + customer: "Contoso Ltd", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "partner", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 869, + status: "paid" + }, + { + id: "ORD-10197", + placedAt: "2024-05-12T15:48:00Z", + customer: "Proseware Inc", + product: "Echo Monitoring", + sku: "MON-550", + region: "LATAM", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 157, + status: "delivered" + }, + { + id: "ORD-10281", + placedAt: "2024-05-12T19:15:00Z", + customer: "City Power & Light", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "direct", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 64, + status: "paid" + }, + { + id: "ORD-10205", + placedAt: "2024-05-12T20:32:00Z", + customer: "Fourth Coffee", + product: "Meridian ETL", + sku: "ETL-404", + region: "EMEA", + channel: "direct", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 732, + status: "delivered" + }, + { + id: "ORD-10085", + placedAt: "2024-05-13T00:31:00Z", + customer: "Blue Yonder Airlines", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "direct", + rep: "Sora Tanaka", + quantity: 7, + unitPrice: 162, + status: "paid" + }, + { + id: "ORD-10373", + placedAt: "2024-05-13T00:58:00Z", + customer: "City Power & Light", + product: "Echo Monitoring", + sku: "MON-550", + region: "North America", + channel: "direct", + rep: "Owen Pratt", + quantity: 5, + unitPrice: 142, + status: "shipped" + }, + { + id: "ORD-10138", + placedAt: "2024-05-13T02:08:00Z", + customer: "City Power & Light", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "partner", + rep: "Sven Olsen", + quantity: 8, + unitPrice: 410, + status: "delivered" + }, + { + id: "ORD-10370", + placedAt: "2024-05-13T02:27:00Z", + customer: "City Power & Light", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "direct", + rep: "Sora Tanaka", + quantity: 3, + unitPrice: 870, + status: "pending" + }, + { + id: "ORD-10005", + placedAt: "2024-05-13T05:43:00Z", + customer: "Margies Travel", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "direct", + rep: "Sven Olsen", + quantity: 5, + unitPrice: 154, + status: "shipped" + }, + { + id: "ORD-10049", + placedAt: "2024-05-13T08:32:00Z", + customer: "Trey Research", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Sven Olsen", + quantity: 1, + unitPrice: 1211, + status: "paid" + }, + { + id: "ORD-10253", + placedAt: "2024-05-13T09:49:00Z", + customer: "Margies Travel", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 727, + status: "delivered" + }, + { + id: "ORD-10178", + placedAt: "2024-05-13T15:00:00Z", + customer: "Litware Inc", + product: "Borealis CRM", + sku: "CRM-210", + region: "North America", + channel: "marketplace", + rep: "Diego Marin", + quantity: 6, + unitPrice: 886, + status: "shipped" + }, + { + id: "ORD-10031", + placedAt: "2024-05-13T16:08:00Z", + customer: "Tailspin Toys", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 845, + status: "delivered" + }, + { + id: "ORD-10257", + placedAt: "2024-05-13T17:06:00Z", + customer: "Tailspin Toys", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Dana Wills", + quantity: 3, + unitPrice: 1180, + status: "paid" + }, + { + id: "ORD-10007", + placedAt: "2024-05-13T18:03:00Z", + customer: "Adventure Works", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 4, + unitPrice: 63, + status: "delivered" + }, + { + id: "ORD-10345", + placedAt: "2024-05-14T01:10:00Z", + customer: "Tailspin Toys", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "direct", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 68, + status: "paid" + }, + { + id: "ORD-10081", + placedAt: "2024-05-14T03:48:00Z", + customer: "Graphic Design Institute", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Priya Nair", + quantity: 8, + unitPrice: 1199, + status: "delivered" + }, + { + id: "ORD-10333", + placedAt: "2024-05-14T05:55:00Z", + customer: "Alpine Ski House", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 8, + unitPrice: 744, + status: "paid" + }, + { + id: "ORD-10051", + placedAt: "2024-05-14T06:54:00Z", + customer: "Northwind Traders", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 2, + unitPrice: 656, + status: "delivered" + }, + { + id: "ORD-10182", + placedAt: "2024-05-14T09:17:00Z", + customer: "Contoso Ltd", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "partner", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 59, + status: "refunded" + }, + { + id: "ORD-10156", + placedAt: "2024-05-14T10:58:00Z", + customer: "Trey Research", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "EMEA", + channel: "direct", + rep: "Diego Marin", + quantity: 7, + unitPrice: 500, + status: "paid" + }, + { + id: "ORD-10032", + placedAt: "2024-05-14T14:27:00Z", + customer: "Graphic Design Institute", + product: "Polaris Reporting", + sku: "RPT-707", + region: "EMEA", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 285, + status: "cancelled" + }, + { + id: "ORD-10186", + placedAt: "2024-05-14T14:28:00Z", + customer: "Fourth Coffee", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "marketplace", + rep: "Diego Marin", + quantity: 4, + unitPrice: 399, + status: "pending" + }, + { + id: "ORD-10169", + placedAt: "2024-05-14T15:55:00Z", + customer: "Adventure Works", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "marketplace", + rep: "Diego Marin", + quantity: 7, + unitPrice: 78, + status: "shipped" + }, + { + id: "ORD-10316", + placedAt: "2024-05-14T16:28:00Z", + customer: "Humongous Insurance", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 7, + unitPrice: 523, + status: "shipped" + }, + { + id: "ORD-10314", + placedAt: "2024-05-14T18:38:00Z", + customer: "Proseware Inc", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "partner", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 420, + status: "cancelled" + }, + { + id: "ORD-10136", + placedAt: "2024-05-14T19:22:00Z", + customer: "Alpine Ski House", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 217, + status: "paid" + }, + { + id: "ORD-10272", + placedAt: "2024-05-14T20:20:00Z", + customer: "Adventure Works", + product: "Polaris Reporting", + sku: "RPT-707", + region: "LATAM", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 1, + unitPrice: 276, + status: "pending" + }, + { + id: "ORD-10363", + placedAt: "2024-05-14T22:03:00Z", + customer: "Fabrikam Inc", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "direct", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 974, + status: "shipped" + }, + { + id: "ORD-10299", + placedAt: "2024-05-14T22:21:00Z", + customer: "Humongous Insurance", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 995, + status: "delivered" + }, + { + id: "ORD-10202", + placedAt: "2024-05-14T23:23:00Z", + customer: "Proseware Inc", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 7, + unitPrice: 403, + status: "cancelled" + }, + { + id: "ORD-10013", + placedAt: "2024-05-14T23:34:00Z", + customer: "Fabrikam Inc", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "direct", + rep: "Diego Marin", + quantity: 1, + unitPrice: 738, + status: "pending" + }, + { + id: "ORD-10122", + placedAt: "2024-05-15T03:55:00Z", + customer: "Graphic Design Institute", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "partner", + rep: "Dana Wills", + quantity: 3, + unitPrice: 404, + status: "cancelled" + }, + { + id: "ORD-10210", + placedAt: "2024-05-15T04:01:00Z", + customer: "Fourth Coffee", + product: "Borealis CRM", + sku: "CRM-210", + region: "North America", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 2, + unitPrice: 905, + status: "shipped" + }, + { + id: "ORD-10029", + placedAt: "2024-05-15T04:05:00Z", + customer: "Alpine Ski House", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 1, + unitPrice: 712, + status: "delivered" + }, + { + id: "ORD-10322", + placedAt: "2024-05-15T07:53:00Z", + customer: "Blue Yonder Airlines", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 896, + status: "delivered" + }, + { + id: "ORD-10209", + placedAt: "2024-05-15T08:38:00Z", + customer: "Adventure Works", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 4, + unitPrice: 1208, + status: "cancelled" + }, + { + id: "ORD-10045", + placedAt: "2024-05-15T10:12:00Z", + customer: "Graphic Design Institute", + product: "Meridian ETL", + sku: "ETL-404", + region: "EMEA", + channel: "partner", + rep: "Dana Wills", + quantity: 5, + unitPrice: 734, + status: "refunded" + }, + { + id: "ORD-10306", + placedAt: "2024-05-15T18:47:00Z", + customer: "Margies Travel", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 6, + unitPrice: 902, + status: "shipped" + }, + { + id: "ORD-10021", + placedAt: "2024-05-15T23:25:00Z", + customer: "Contoso Ltd", + product: "Echo Monitoring", + sku: "MON-550", + region: "LATAM", + channel: "self-serve", + rep: "Diego Marin", + quantity: 2, + unitPrice: 142, + status: "delivered" + }, + { + id: "ORD-10011", + placedAt: "2024-05-16T00:55:00Z", + customer: "Graphic Design Institute", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "direct", + rep: "Priya Nair", + quantity: 8, + unitPrice: 972, + status: "pending" + }, + { + id: "ORD-10334", + placedAt: "2024-05-16T00:55:00Z", + customer: "City Power & Light", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "direct", + rep: "Mateo Russo", + quantity: 6, + unitPrice: 1113, + status: "delivered" + }, + { + id: "ORD-10222", + placedAt: "2024-05-16T02:01:00Z", + customer: "School of Fine Art", + product: "Nimbus Compute", + sku: "CMP-505", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 2, + unitPrice: 1109, + status: "delivered" + }, + { + id: "ORD-10116", + placedAt: "2024-05-16T02:04:00Z", + customer: "Wide World Importers", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 2, + unitPrice: 300, + status: "paid" + }, + { + id: "ORD-10026", + placedAt: "2024-05-16T05:17:00Z", + customer: "Proseware Inc", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 2, + unitPrice: 393, + status: "shipped" + }, + { + id: "ORD-10009", + placedAt: "2024-05-16T06:11:00Z", + customer: "Coho Vineyard", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 46, + status: "delivered" + }, + { + id: "ORD-10155", + placedAt: "2024-05-16T07:24:00Z", + customer: "Fabrikam Inc", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "APAC", + channel: "direct", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 983, + status: "delivered" + }, + { + id: "ORD-10102", + placedAt: "2024-05-16T10:39:00Z", + customer: "Lucerne Publishing", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "direct", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 90, + status: "delivered" + }, + { + id: "ORD-10302", + placedAt: "2024-05-16T12:23:00Z", + customer: "Humongous Insurance", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 1110, + status: "refunded" + }, + { + id: "ORD-10237", + placedAt: "2024-05-16T15:29:00Z", + customer: "Northwind Traders", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "direct", + rep: "Owen Pratt", + quantity: 6, + unitPrice: 739, + status: "paid" + }, + { + id: "ORD-10046", + placedAt: "2024-05-16T15:57:00Z", + customer: "Tailspin Toys", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "partner", + rep: "Owen Pratt", + quantity: 6, + unitPrice: 1113, + status: "paid" + }, + { + id: "ORD-10163", + placedAt: "2024-05-16T16:50:00Z", + customer: "Fourth Coffee", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "LATAM", + channel: "direct", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 635, + status: "delivered" + }, + { + id: "ORD-10196", + placedAt: "2024-05-16T17:46:00Z", + customer: "Fabrikam Inc", + product: "Delta Insights", + sku: "INS-440", + region: "North America", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 2, + unitPrice: 332, + status: "paid" + }, + { + id: "ORD-10024", + placedAt: "2024-05-16T18:20:00Z", + customer: "Litware Inc", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "partner", + rep: "Sven Olsen", + quantity: 6, + unitPrice: 229, + status: "cancelled" + }, + { + id: "ORD-10339", + placedAt: "2024-05-16T20:20:00Z", + customer: "Adventure Works", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 2, + unitPrice: 659, + status: "pending" + }, + { + id: "ORD-10082", + placedAt: "2024-05-16T23:02:00Z", + customer: "Humongous Insurance", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "partner", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 890, + status: "delivered" + }, + { + id: "ORD-10274", + placedAt: "2024-05-16T23:56:00Z", + customer: "Contoso Ltd", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "marketplace", + rep: "Sven Olsen", + quantity: 8, + unitPrice: 874, + status: "pending" + }, + { + id: "ORD-10002", + placedAt: "2024-05-17T04:16:00Z", + customer: "Wide World Importers", + product: "Borealis CRM", + sku: "CRM-210", + region: "North America", + channel: "direct", + rep: "Priya Nair", + quantity: 2, + unitPrice: 908, + status: "shipped" + }, + { + id: "ORD-10170", + placedAt: "2024-05-17T05:32:00Z", + customer: "Trey Research", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "partner", + rep: "Dana Wills", + quantity: 1, + unitPrice: 402, + status: "delivered" + }, + { + id: "ORD-10086", + placedAt: "2024-05-17T05:54:00Z", + customer: "City Power & Light", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 68, + status: "cancelled" + }, + { + id: "ORD-10288", + placedAt: "2024-05-17T11:21:00Z", + customer: "Alpine Ski House", + product: "Polaris Reporting", + sku: "RPT-707", + region: "LATAM", + channel: "partner", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 298, + status: "shipped" + }, + { + id: "ORD-10106", + placedAt: "2024-05-17T13:49:00Z", + customer: "Blue Yonder Airlines", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "partner", + rep: "Sora Tanaka", + quantity: 1, + unitPrice: 410, + status: "shipped" + }, + { + id: "ORD-10201", + placedAt: "2024-05-17T14:23:00Z", + customer: "Coho Vineyard", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "marketplace", + rep: "Dana Wills", + quantity: 4, + unitPrice: 71, + status: "shipped" + }, + { + id: "ORD-10160", + placedAt: "2024-05-17T18:43:00Z", + customer: "Wide World Importers", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "direct", + rep: "Hugo Bernard", + quantity: 8, + unitPrice: 309, + status: "delivered" + }, + { + id: "ORD-10187", + placedAt: "2024-05-17T21:51:00Z", + customer: "Humongous Insurance", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "self-serve", + rep: "Dana Wills", + quantity: 3, + unitPrice: 987, + status: "delivered" + }, + { + id: "ORD-10359", + placedAt: "2024-05-17T22:44:00Z", + customer: "City Power & Light", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 8, + unitPrice: 25, + status: "shipped" + }, + { + id: "ORD-10078", + placedAt: "2024-05-17T23:04:00Z", + customer: "Wide World Importers", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "self-serve", + rep: "Diego Marin", + quantity: 1, + unitPrice: 1093, + status: "paid" + }, + { + id: "ORD-10071", + placedAt: "2024-05-18T00:32:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 57, + status: "shipped" + }, + { + id: "ORD-10348", + placedAt: "2024-05-18T04:23:00Z", + customer: "School of Fine Art", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 538, + status: "delivered" + }, + { + id: "ORD-10093", + placedAt: "2024-05-18T08:16:00Z", + customer: "Litware Inc", + product: "Meridian ETL", + sku: "ETL-404", + region: "EMEA", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 3, + unitPrice: 728, + status: "shipped" + }, + { + id: "ORD-10114", + placedAt: "2024-05-18T08:56:00Z", + customer: "Tailspin Toys", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "partner", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 893, + status: "shipped" + }, + { + id: "ORD-10019", + placedAt: "2024-05-18T09:30:00Z", + customer: "Northwind Traders", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 4, + unitPrice: 658, + status: "refunded" + }, + { + id: "ORD-10321", + placedAt: "2024-05-18T12:26:00Z", + customer: "Alpine Ski House", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "LATAM", + channel: "marketplace", + rep: "Mateo Russo", + quantity: 7, + unitPrice: 1203, + status: "delivered" + }, + { + id: "ORD-10292", + placedAt: "2024-05-18T13:07:00Z", + customer: "Northwind Traders", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "direct", + rep: "Sven Olsen", + quantity: 7, + unitPrice: 312, + status: "shipped" + }, + { + id: "ORD-10055", + placedAt: "2024-05-18T14:49:00Z", + customer: "Proseware Inc", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 62, + status: "delivered" + }, + { + id: "ORD-10158", + placedAt: "2024-05-18T15:39:00Z", + customer: "Northwind Traders", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 1113, + status: "shipped" + }, + { + id: "ORD-10150", + placedAt: "2024-05-18T19:45:00Z", + customer: "City Power & Light", + product: "Fjord Storage", + sku: "STO-660", + region: "LATAM", + channel: "direct", + rep: "Sora Tanaka", + quantity: 7, + unitPrice: 68, + status: "shipped" + }, + { + id: "ORD-10331", + placedAt: "2024-05-18T20:29:00Z", + customer: "Coho Vineyard", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 978, + status: "paid" + }, + { + id: "ORD-10072", + placedAt: "2024-05-19T02:24:00Z", + customer: "Proseware Inc", + product: "Helix Identity", + sku: "IDN-880", + region: "LATAM", + channel: "partner", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 235, + status: "refunded" + }, + { + id: "ORD-10035", + placedAt: "2024-05-19T02:52:00Z", + customer: "School of Fine Art", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 1, + unitPrice: 621, + status: "delivered" + }, + { + id: "ORD-10326", + placedAt: "2024-05-19T02:58:00Z", + customer: "Alpine Ski House", + product: "Fjord Storage", + sku: "STO-660", + region: "LATAM", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 70, + status: "pending" + }, + { + id: "ORD-10145", + placedAt: "2024-05-19T03:36:00Z", + customer: "Wingtip Toys", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "LATAM", + channel: "partner", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 1200, + status: "refunded" + }, + { + id: "ORD-10276", + placedAt: "2024-05-19T04:22:00Z", + customer: "City Power & Light", + product: "Delta Insights", + sku: "INS-440", + region: "North America", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 339, + status: "cancelled" + }, + { + id: "ORD-10250", + placedAt: "2024-05-19T04:52:00Z", + customer: "School of Fine Art", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 3, + unitPrice: 416, + status: "delivered" + }, + { + id: "ORD-10264", + placedAt: "2024-05-19T07:33:00Z", + customer: "Northwind Traders", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "marketplace", + rep: "Priya Nair", + quantity: 5, + unitPrice: 213, + status: "refunded" + }, + { + id: "ORD-10130", + placedAt: "2024-05-19T11:38:00Z", + customer: "Lucerne Publishing", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "self-serve", + rep: "Diego Marin", + quantity: 7, + unitPrice: 876, + status: "pending" + }, + { + id: "ORD-10181", + placedAt: "2024-05-19T12:32:00Z", + customer: "Coho Vineyard", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "direct", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 141, + status: "pending" + }, + { + id: "ORD-10228", + placedAt: "2024-05-19T13:22:00Z", + customer: "Proseware Inc", + product: "Delta Insights", + sku: "INS-440", + region: "North America", + channel: "partner", + rep: "Dana Wills", + quantity: 4, + unitPrice: 334, + status: "delivered" + }, + { + id: "ORD-10206", + placedAt: "2024-05-19T13:57:00Z", + customer: "Alpine Ski House", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "direct", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 1114, + status: "refunded" + }, + { + id: "ORD-10364", + placedAt: "2024-05-19T20:03:00Z", + customer: "School of Fine Art", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "partner", + rep: "Diego Marin", + quantity: 5, + unitPrice: 527, + status: "paid" + }, + { + id: "ORD-10132", + placedAt: "2024-05-19T22:57:00Z", + customer: "Fabrikam Inc", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "partner", + rep: "Lena Fischer", + quantity: 8, + unitPrice: 337, + status: "paid" + }, + { + id: "ORD-10242", + placedAt: "2024-05-20T00:28:00Z", + customer: "Alpine Ski House", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "direct", + rep: "Sora Tanaka", + quantity: 8, + unitPrice: 874, + status: "shipped" + }, + { + id: "ORD-10099", + placedAt: "2024-05-20T01:51:00Z", + customer: "Contoso Ltd", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "EMEA", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 650, + status: "paid" + }, + { + id: "ORD-10248", + placedAt: "2024-05-20T01:53:00Z", + customer: "Trey Research", + product: "Helix Identity", + sku: "IDN-880", + region: "LATAM", + channel: "partner", + rep: "Mateo Russo", + quantity: 7, + unitPrice: 215, + status: "delivered" + }, + { + id: "ORD-10113", + placedAt: "2024-05-20T02:06:00Z", + customer: "Contoso Ltd", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Diego Marin", + quantity: 1, + unitPrice: 1201, + status: "pending" + }, + { + id: "ORD-10287", + placedAt: "2024-05-20T02:20:00Z", + customer: "Blue Yonder Airlines", + product: "Onyx Security", + sku: "SEC-606", + region: "EMEA", + channel: "partner", + rep: "Diego Marin", + quantity: 8, + unitPrice: 864, + status: "shipped" + }, + { + id: "ORD-10350", + placedAt: "2024-05-20T06:01:00Z", + customer: "Margies Travel", + product: "Nimbus Compute", + sku: "CMP-505", + region: "APAC", + channel: "direct", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 1085, + status: "refunded" + }, + { + id: "ORD-10353", + placedAt: "2024-05-20T11:32:00Z", + customer: "Coho Vineyard", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "EMEA", + channel: "direct", + rep: "Dana Wills", + quantity: 1, + unitPrice: 1195, + status: "delivered" + }, + { + id: "ORD-10137", + placedAt: "2024-05-20T11:52:00Z", + customer: "Northwind Traders", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "self-serve", + rep: "Dana Wills", + quantity: 7, + unitPrice: 70, + status: "pending" + }, + { + id: "ORD-10069", + placedAt: "2024-05-20T14:29:00Z", + customer: "Litware Inc", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "direct", + rep: "Sora Tanaka", + quantity: 7, + unitPrice: 165, + status: "delivered" + }, + { + id: "ORD-10015", + placedAt: "2024-05-20T14:34:00Z", + customer: "Tailspin Toys", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 868, + status: "paid" + }, + { + id: "ORD-10217", + placedAt: "2024-05-20T14:41:00Z", + customer: "Northwind Traders", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "direct", + rep: "Priya Nair", + quantity: 8, + unitPrice: 62, + status: "shipped" + }, + { + id: "ORD-10214", + placedAt: "2024-05-20T22:10:00Z", + customer: "Lucerne Publishing", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "marketplace", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 68, + status: "shipped" + }, + { + id: "ORD-10265", + placedAt: "2024-05-20T22:31:00Z", + customer: "Proseware Inc", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "partner", + rep: "Dana Wills", + quantity: 1, + unitPrice: 58, + status: "shipped" + }, + { + id: "ORD-10195", + placedAt: "2024-05-21T02:12:00Z", + customer: "Proseware Inc", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "LATAM", + channel: "partner", + rep: "Aisha Khan", + quantity: 6, + unitPrice: 645, + status: "delivered" + }, + { + id: "ORD-10225", + placedAt: "2024-05-21T07:35:00Z", + customer: "Fabrikam Inc", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 1189, + status: "delivered" + }, + { + id: "ORD-10180", + placedAt: "2024-05-21T08:15:00Z", + customer: "Wingtip Toys", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "direct", + rep: "Owen Pratt", + quantity: 4, + unitPrice: 337, + status: "shipped" + }, + { + id: "ORD-10047", + placedAt: "2024-05-21T09:50:00Z", + customer: "Wide World Importers", + product: "Onyx Security", + sku: "SEC-606", + region: "EMEA", + channel: "direct", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 855, + status: "shipped" + }, + { + id: "ORD-10315", + placedAt: "2024-05-21T11:07:00Z", + customer: "Wide World Importers", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 975, + status: "delivered" + }, + { + id: "ORD-10308", + placedAt: "2024-05-21T14:41:00Z", + customer: "Coho Vineyard", + product: "Delta Insights", + sku: "INS-440", + region: "North America", + channel: "marketplace", + rep: "Priya Nair", + quantity: 2, + unitPrice: 339, + status: "refunded" + }, + { + id: "ORD-10352", + placedAt: "2024-05-21T14:46:00Z", + customer: "Alpine Ski House", + product: "Polaris Reporting", + sku: "RPT-707", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 271, + status: "delivered" + }, + { + id: "ORD-10270", + placedAt: "2024-05-21T15:15:00Z", + customer: "Fourth Coffee", + product: "Nimbus Compute", + sku: "CMP-505", + region: "APAC", + channel: "direct", + rep: "Diego Marin", + quantity: 3, + unitPrice: 1089, + status: "cancelled" + }, + { + id: "ORD-10324", + placedAt: "2024-05-21T15:21:00Z", + customer: "Tailspin Toys", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 320, + status: "delivered" + }, + { + id: "ORD-10291", + placedAt: "2024-05-21T17:15:00Z", + customer: "Graphic Design Institute", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "APAC", + channel: "marketplace", + rep: "Priya Nair", + quantity: 8, + unitPrice: 629, + status: "delivered" + }, + { + id: "ORD-10054", + placedAt: "2024-05-21T19:36:00Z", + customer: "Wide World Importers", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "direct", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 88, + status: "cancelled" + }, + { + id: "ORD-10227", + placedAt: "2024-05-22T00:21:00Z", + customer: "Wingtip Toys", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "self-serve", + rep: "Dana Wills", + quantity: 8, + unitPrice: 640, + status: "pending" + }, + { + id: "ORD-10213", + placedAt: "2024-05-22T04:25:00Z", + customer: "Margies Travel", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 161, + status: "shipped" + }, + { + id: "ORD-10246", + placedAt: "2024-05-22T08:25:00Z", + customer: "Tailspin Toys", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 4, + unitPrice: 64, + status: "paid" + }, + { + id: "ORD-10294", + placedAt: "2024-05-22T08:30:00Z", + customer: "Lucerne Publishing", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 87, + status: "pending" + }, + { + id: "ORD-10188", + placedAt: "2024-05-22T10:03:00Z", + customer: "Tailspin Toys", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "marketplace", + rep: "Priya Nair", + quantity: 3, + unitPrice: 536, + status: "delivered" + }, + { + id: "ORD-10075", + placedAt: "2024-05-22T10:04:00Z", + customer: "Fourth Coffee", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 962, + status: "paid" + }, + { + id: "ORD-10312", + placedAt: "2024-05-22T10:13:00Z", + customer: "Trey Research", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "partner", + rep: "Sven Olsen", + quantity: 7, + unitPrice: 215, + status: "shipped" + }, + { + id: "ORD-10165", + placedAt: "2024-05-22T11:39:00Z", + customer: "Margies Travel", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "partner", + rep: "Hugo Bernard", + quantity: 1, + unitPrice: 157, + status: "delivered" + }, + { + id: "ORD-10318", + placedAt: "2024-05-22T13:57:00Z", + customer: "Margies Travel", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 1114, + status: "cancelled" + }, + { + id: "ORD-10190", + placedAt: "2024-05-22T15:49:00Z", + customer: "Lucerne Publishing", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 2, + unitPrice: 1112, + status: "cancelled" + }, + { + id: "ORD-10100", + placedAt: "2024-05-22T18:37:00Z", + customer: "School of Fine Art", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 5, + unitPrice: 334, + status: "delivered" + }, + { + id: "ORD-10010", + placedAt: "2024-05-22T23:45:00Z", + customer: "Wide World Importers", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "partner", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 417, + status: "paid" + }, + { + id: "ORD-10236", + placedAt: "2024-05-23T00:12:00Z", + customer: "Contoso Ltd", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "direct", + rep: "Diego Marin", + quantity: 5, + unitPrice: 506, + status: "delivered" + }, + { + id: "ORD-10261", + placedAt: "2024-05-23T00:43:00Z", + customer: "Tailspin Toys", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 2, + unitPrice: 139, + status: "shipped" + }, + { + id: "ORD-10309", + placedAt: "2024-05-23T01:47:00Z", + customer: "Wingtip Toys", + product: "Echo Monitoring", + sku: "MON-550", + region: "LATAM", + channel: "self-serve", + rep: "Diego Marin", + quantity: 7, + unitPrice: 150, + status: "shipped" + }, + { + id: "ORD-10105", + placedAt: "2024-05-23T01:54:00Z", + customer: "Wide World Importers", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "direct", + rep: "Lena Fischer", + quantity: 5, + unitPrice: 61, + status: "shipped" + }, + { + id: "ORD-10104", + placedAt: "2024-05-23T02:03:00Z", + customer: "Wide World Importers", + product: "Helix Identity", + sku: "IDN-880", + region: "LATAM", + channel: "partner", + rep: "Owen Pratt", + quantity: 2, + unitPrice: 201, + status: "paid" + }, + { + id: "ORD-10157", + placedAt: "2024-05-23T08:09:00Z", + customer: "Northwind Traders", + product: "Meridian ETL", + sku: "ETL-404", + region: "LATAM", + channel: "partner", + rep: "Diego Marin", + quantity: 7, + unitPrice: 723, + status: "shipped" + }, + { + id: "ORD-10208", + placedAt: "2024-05-23T08:34:00Z", + customer: "Northwind Traders", + product: "Polaris Reporting", + sku: "RPT-707", + region: "APAC", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 286, + status: "pending" + }, + { + id: "ORD-10096", + placedAt: "2024-05-23T09:41:00Z", + customer: "Coho Vineyard", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 2, + unitPrice: 283, + status: "paid" + }, + { + id: "ORD-10374", + placedAt: "2024-05-23T10:17:00Z", + customer: "Coho Vineyard", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 76, + status: "delivered" + }, + { + id: "ORD-10249", + placedAt: "2024-05-23T12:41:00Z", + customer: "Wingtip Toys", + product: "Ion Messaging", + sku: "MSG-990", + region: "APAC", + channel: "marketplace", + rep: "Diego Marin", + quantity: 1, + unitPrice: 62, + status: "shipped" + }, + { + id: "ORD-10296", + placedAt: "2024-05-23T13:39:00Z", + customer: "Wingtip Toys", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "marketplace", + rep: "Diego Marin", + quantity: 2, + unitPrice: 222, + status: "paid" + }, + { + id: "ORD-10129", + placedAt: "2024-05-23T14:38:00Z", + customer: "Contoso Ltd", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "partner", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 1181, + status: "shipped" + }, + { + id: "ORD-10135", + placedAt: "2024-05-23T15:31:00Z", + customer: "Fabrikam Inc", + product: "Glacier Backup", + sku: "BAK-770", + region: "LATAM", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 7, + unitPrice: 32, + status: "refunded" + }, + { + id: "ORD-10154", + placedAt: "2024-05-23T22:44:00Z", + customer: "Blue Yonder Airlines", + product: "Juniper Workflow", + sku: "WFL-101", + region: "LATAM", + channel: "partner", + rep: "Dana Wills", + quantity: 1, + unitPrice: 403, + status: "shipped" + }, + { + id: "ORD-10307", + placedAt: "2024-05-23T23:25:00Z", + customer: "Fabrikam Inc", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "APAC", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 8, + unitPrice: 636, + status: "pending" + }, + { + id: "ORD-10103", + placedAt: "2024-05-23T23:51:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "marketplace", + rep: "Priya Nair", + quantity: 5, + unitPrice: 44, + status: "delivered" + }, + { + id: "ORD-10198", + placedAt: "2024-05-24T00:34:00Z", + customer: "City Power & Light", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "partner", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 67, + status: "paid" + }, + { + id: "ORD-10053", + placedAt: "2024-05-24T05:36:00Z", + customer: "Proseware Inc", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "direct", + rep: "Diego Marin", + quantity: 7, + unitPrice: 155, + status: "cancelled" + }, + { + id: "ORD-10342", + placedAt: "2024-05-24T12:10:00Z", + customer: "Litware Inc", + product: "Fjord Storage", + sku: "STO-660", + region: "EMEA", + channel: "direct", + rep: "Sven Olsen", + quantity: 8, + unitPrice: 58, + status: "delivered" + }, + { + id: "ORD-10379", + placedAt: "2024-05-24T12:33:00Z", + customer: "City Power & Light", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "LATAM", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 974, + status: "delivered" + }, + { + id: "ORD-10042", + placedAt: "2024-05-24T12:49:00Z", + customer: "Northwind Traders", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 403, + status: "shipped" + }, + { + id: "ORD-10317", + placedAt: "2024-05-24T13:49:00Z", + customer: "Fourth Coffee", + product: "Meridian ETL", + sku: "ETL-404", + region: "North America", + channel: "marketplace", + rep: "Dana Wills", + quantity: 8, + unitPrice: 713, + status: "delivered" + }, + { + id: "ORD-10357", + placedAt: "2024-05-24T15:20:00Z", + customer: "Fourth Coffee", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "direct", + rep: "Sven Olsen", + quantity: 1, + unitPrice: 134, + status: "refunded" + }, + { + id: "ORD-10092", + placedAt: "2024-05-24T15:52:00Z", + customer: "Wingtip Toys", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 1, + unitPrice: 512, + status: "delivered" + }, + { + id: "ORD-10095", + placedAt: "2024-05-24T16:06:00Z", + customer: "Humongous Insurance", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 5, + unitPrice: 862, + status: "pending" + }, + { + id: "ORD-10226", + placedAt: "2024-05-24T18:48:00Z", + customer: "Wingtip Toys", + product: "Borealis CRM", + sku: "CRM-210", + region: "EMEA", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 890, + status: "pending" + }, + { + id: "ORD-10003", + placedAt: "2024-05-24T22:27:00Z", + customer: "Litware Inc", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "partner", + rep: "Hugo Bernard", + quantity: 2, + unitPrice: 650, + status: "paid" + }, + { + id: "ORD-10365", + placedAt: "2024-05-25T00:27:00Z", + customer: "Blue Yonder Airlines", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 3, + unitPrice: 732, + status: "pending" + }, + { + id: "ORD-10325", + placedAt: "2024-05-25T01:59:00Z", + customer: "Wingtip Toys", + product: "Echo Monitoring", + sku: "MON-550", + region: "EMEA", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 155, + status: "refunded" + }, + { + id: "ORD-10143", + placedAt: "2024-05-25T03:36:00Z", + customer: "Lucerne Publishing", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 857, + status: "delivered" + }, + { + id: "ORD-10134", + placedAt: "2024-05-25T04:25:00Z", + customer: "Humongous Insurance", + product: "Fjord Storage", + sku: "STO-660", + region: "LATAM", + channel: "self-serve", + rep: "Priya Nair", + quantity: 3, + unitPrice: 57, + status: "paid" + }, + { + id: "ORD-10301", + placedAt: "2024-05-25T04:58:00Z", + customer: "City Power & Light", + product: "Meridian ETL", + sku: "ETL-404", + region: "North America", + channel: "self-serve", + rep: "Sora Tanaka", + quantity: 2, + unitPrice: 724, + status: "refunded" + }, + { + id: "ORD-10244", + placedAt: "2024-05-25T05:20:00Z", + customer: "Proseware Inc", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "partner", + rep: "Lena Fischer", + quantity: 5, + unitPrice: 326, + status: "pending" + }, + { + id: "ORD-10061", + placedAt: "2024-05-25T16:28:00Z", + customer: "Adventure Works", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 8, + unitPrice: 720, + status: "shipped" + }, + { + id: "ORD-10087", + placedAt: "2024-05-25T16:41:00Z", + customer: "Adventure Works", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "self-serve", + rep: "Diego Marin", + quantity: 7, + unitPrice: 40, + status: "shipped" + }, + { + id: "ORD-10001", + placedAt: "2024-05-25T19:39:00Z", + customer: "Fabrikam Inc", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "North America", + channel: "self-serve", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 1195, + status: "cancelled" + }, + { + id: "ORD-10030", + placedAt: "2024-05-25T21:06:00Z", + customer: "Litware Inc", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "marketplace", + rep: "Sven Olsen", + quantity: 3, + unitPrice: 1115, + status: "cancelled" + }, + { + id: "ORD-10115", + placedAt: "2024-05-25T21:27:00Z", + customer: "Lucerne Publishing", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "LATAM", + channel: "direct", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 649, + status: "refunded" + }, + { + id: "ORD-10336", + placedAt: "2024-05-25T22:58:00Z", + customer: "Tailspin Toys", + product: "Polaris Reporting", + sku: "RPT-707", + region: "APAC", + channel: "direct", + rep: "Aisha Khan", + quantity: 3, + unitPrice: 281, + status: "paid" + }, + { + id: "ORD-10216", + placedAt: "2024-05-25T23:02:00Z", + customer: "Graphic Design Institute", + product: "Helix Identity", + sku: "IDN-880", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 6, + unitPrice: 228, + status: "delivered" + }, + { + id: "ORD-10080", + placedAt: "2024-05-26T01:16:00Z", + customer: "Proseware Inc", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 6, + unitPrice: 289, + status: "pending" + }, + { + id: "ORD-10161", + placedAt: "2024-05-26T02:41:00Z", + customer: "School of Fine Art", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "LATAM", + channel: "direct", + rep: "Owen Pratt", + quantity: 8, + unitPrice: 1185, + status: "pending" + }, + { + id: "ORD-10343", + placedAt: "2024-05-26T08:07:00Z", + customer: "Wingtip Toys", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "self-serve", + rep: "Mateo Russo", + quantity: 2, + unitPrice: 45, + status: "pending" + }, + { + id: "ORD-10077", + placedAt: "2024-05-26T08:37:00Z", + customer: "Contoso Ltd", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 5, + unitPrice: 727, + status: "delivered" + }, + { + id: "ORD-10327", + placedAt: "2024-05-26T09:58:00Z", + customer: "School of Fine Art", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "self-serve", + rep: "Priya Nair", + quantity: 3, + unitPrice: 39, + status: "pending" + }, + { + id: "ORD-10057", + placedAt: "2024-05-26T10:39:00Z", + customer: "Northwind Traders", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "partner", + rep: "Dana Wills", + quantity: 7, + unitPrice: 53, + status: "shipped" + }, + { + id: "ORD-10088", + placedAt: "2024-05-26T16:48:00Z", + customer: "Coho Vineyard", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "marketplace", + rep: "Diego Marin", + quantity: 5, + unitPrice: 218, + status: "shipped" + }, + { + id: "ORD-10286", + placedAt: "2024-05-26T21:13:00Z", + customer: "Wingtip Toys", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "direct", + rep: "Diego Marin", + quantity: 2, + unitPrice: 1092, + status: "shipped" + }, + { + id: "ORD-10140", + placedAt: "2024-05-26T22:31:00Z", + customer: "Tailspin Toys", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "North America", + channel: "direct", + rep: "Mateo Russo", + quantity: 5, + unitPrice: 520, + status: "paid" + }, + { + id: "ORD-10254", + placedAt: "2024-05-26T23:02:00Z", + customer: "Northwind Traders", + product: "Nimbus Compute", + sku: "CMP-505", + region: "EMEA", + channel: "self-serve", + rep: "Sven Olsen", + quantity: 3, + unitPrice: 1099, + status: "delivered" + }, + { + id: "ORD-10108", + placedAt: "2024-05-26T23:31:00Z", + customer: "Blue Yonder Airlines", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "APAC", + channel: "partner", + rep: "Priya Nair", + quantity: 6, + unitPrice: 504, + status: "paid" + }, + { + id: "ORD-10303", + placedAt: "2024-05-27T00:21:00Z", + customer: "Fourth Coffee", + product: "Onyx Security", + sku: "SEC-606", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 6, + unitPrice: 855, + status: "shipped" + }, + { + id: "ORD-10258", + placedAt: "2024-05-27T04:22:00Z", + customer: "Alpine Ski House", + product: "Borealis CRM", + sku: "CRM-210", + region: "LATAM", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 3, + unitPrice: 908, + status: "delivered" + }, + { + id: "ORD-10267", + placedAt: "2024-05-27T06:00:00Z", + customer: "Coho Vineyard", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 3, + unitPrice: 966, + status: "delivered" + }, + { + id: "ORD-10179", + placedAt: "2024-05-27T06:50:00Z", + customer: "City Power & Light", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "North America", + channel: "marketplace", + rep: "Dana Wills", + quantity: 5, + unitPrice: 644, + status: "paid" + }, + { + id: "ORD-10184", + placedAt: "2024-05-27T10:30:00Z", + customer: "Tailspin Toys", + product: "Helix Identity", + sku: "IDN-880", + region: "North America", + channel: "partner", + rep: "Diego Marin", + quantity: 5, + unitPrice: 208, + status: "pending" + }, + { + id: "ORD-10023", + placedAt: "2024-05-27T12:22:00Z", + customer: "City Power & Light", + product: "Glacier Backup", + sku: "BAK-770", + region: "APAC", + channel: "direct", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 31, + status: "pending" + }, + { + id: "ORD-10037", + placedAt: "2024-05-27T16:22:00Z", + customer: "Alpine Ski House", + product: "Echo Monitoring", + sku: "MON-550", + region: "APAC", + channel: "marketplace", + rep: "Priya Nair", + quantity: 3, + unitPrice: 141, + status: "refunded" + }, + { + id: "ORD-10247", + placedAt: "2024-05-27T22:46:00Z", + customer: "Trey Research", + product: "Glacier Backup", + sku: "BAK-770", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 7, + unitPrice: 60, + status: "delivered" + }, + { + id: "ORD-10027", + placedAt: "2024-05-27T23:51:00Z", + customer: "Fabrikam Inc", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "North America", + channel: "partner", + rep: "Sven Olsen", + quantity: 3, + unitPrice: 979, + status: "cancelled" + }, + { + id: "ORD-10064", + placedAt: "2024-05-28T02:28:00Z", + customer: "Fabrikam Inc", + product: "Polaris Reporting", + sku: "RPT-707", + region: "APAC", + channel: "self-serve", + rep: "Priya Nair", + quantity: 7, + unitPrice: 273, + status: "delivered" + }, + { + id: "ORD-10022", + placedAt: "2024-05-28T03:38:00Z", + customer: "Fourth Coffee", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "partner", + rep: "Sora Tanaka", + quantity: 4, + unitPrice: 56, + status: "refunded" + }, + { + id: "ORD-10014", + placedAt: "2024-05-28T05:33:00Z", + customer: "Adventure Works", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "direct", + rep: "Owen Pratt", + quantity: 6, + unitPrice: 1100, + status: "paid" + }, + { + id: "ORD-10074", + placedAt: "2024-05-28T05:54:00Z", + customer: "Margies Travel", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 2, + unitPrice: 419, + status: "delivered" + }, + { + id: "ORD-10147", + placedAt: "2024-05-28T07:03:00Z", + customer: "Lucerne Publishing", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 1, + unitPrice: 652, + status: "delivered" + }, + { + id: "ORD-10371", + placedAt: "2024-05-28T07:48:00Z", + customer: "City Power & Light", + product: "Cascade Data Pipeline", + sku: "PIPE-330", + region: "APAC", + channel: "self-serve", + rep: "Dana Wills", + quantity: 8, + unitPrice: 655, + status: "shipped" + }, + { + id: "ORD-10375", + placedAt: "2024-05-28T08:26:00Z", + customer: "Fabrikam Inc", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 2, + unitPrice: 43, + status: "delivered" + }, + { + id: "ORD-10118", + placedAt: "2024-05-28T08:32:00Z", + customer: "Humongous Insurance", + product: "Fjord Storage", + sku: "STO-660", + region: "North America", + channel: "marketplace", + rep: "Diego Marin", + quantity: 1, + unitPrice: 81, + status: "paid" + }, + { + id: "ORD-10065", + placedAt: "2024-05-28T08:58:00Z", + customer: "Graphic Design Institute", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "EMEA", + channel: "direct", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 1203, + status: "delivered" + }, + { + id: "ORD-10289", + placedAt: "2024-05-28T10:03:00Z", + customer: "Lucerne Publishing", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "EMEA", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 1199, + status: "refunded" + }, + { + id: "ORD-10252", + placedAt: "2024-05-28T11:06:00Z", + customer: "Alpine Ski House", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "EMEA", + channel: "partner", + rep: "Priya Nair", + quantity: 8, + unitPrice: 513, + status: "shipped" + }, + { + id: "ORD-10119", + placedAt: "2024-05-28T12:50:00Z", + customer: "Lucerne Publishing", + product: "Glacier Backup", + sku: "BAK-770", + region: "North America", + channel: "marketplace", + rep: "Sora Tanaka", + quantity: 1, + unitPrice: 41, + status: "pending" + }, + { + id: "ORD-10016", + placedAt: "2024-05-28T13:48:00Z", + customer: "School of Fine Art", + product: "Polaris Reporting", + sku: "RPT-707", + region: "North America", + channel: "partner", + rep: "Lena Fischer", + quantity: 2, + unitPrice: 283, + status: "shipped" + }, + { + id: "ORD-10168", + placedAt: "2024-05-28T14:05:00Z", + customer: "Tailspin Toys", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 7, + unitPrice: 213, + status: "shipped" + }, + { + id: "ORD-10094", + placedAt: "2024-05-28T15:55:00Z", + customer: "Alpine Ski House", + product: "Nimbus Compute", + sku: "CMP-505", + region: "APAC", + channel: "self-serve", + rep: "Hugo Bernard", + quantity: 8, + unitPrice: 1086, + status: "delivered" + }, + { + id: "ORD-10036", + placedAt: "2024-05-28T19:44:00Z", + customer: "Trey Research", + product: "Delta Insights", + sku: "INS-440", + region: "LATAM", + channel: "self-serve", + rep: "Dana Wills", + quantity: 2, + unitPrice: 327, + status: "shipped" + }, + { + id: "ORD-10335", + placedAt: "2024-05-28T20:08:00Z", + customer: "Contoso Ltd", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "direct", + rep: "Diego Marin", + quantity: 6, + unitPrice: 841, + status: "delivered" + }, + { + id: "ORD-10025", + placedAt: "2024-05-28T20:19:00Z", + customer: "Proseware Inc", + product: "Ion Messaging", + sku: "MSG-990", + region: "North America", + channel: "self-serve", + rep: "Priya Nair", + quantity: 1, + unitPrice: 59, + status: "shipped" + }, + { + id: "ORD-10126", + placedAt: "2024-05-28T22:23:00Z", + customer: "Wingtip Toys", + product: "Nimbus Compute", + sku: "CMP-505", + region: "North America", + channel: "partner", + rep: "Priya Nair", + quantity: 5, + unitPrice: 1110, + status: "delivered" + }, + { + id: "ORD-10112", + placedAt: "2024-05-28T22:43:00Z", + customer: "Lucerne Publishing", + product: "Polaris Reporting", + sku: "RPT-707", + region: "EMEA", + channel: "partner", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 285, + status: "pending" + }, + { + id: "ORD-10068", + placedAt: "2024-05-28T23:49:00Z", + customer: "Fabrikam Inc", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "direct", + rep: "Owen Pratt", + quantity: 8, + unitPrice: 305, + status: "delivered" + }, + { + id: "ORD-10310", + placedAt: "2024-05-29T00:02:00Z", + customer: "Wingtip Toys", + product: "Fjord Storage", + sku: "STO-660", + region: "APAC", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 92, + status: "paid" + }, + { + id: "ORD-10232", + placedAt: "2024-05-29T02:04:00Z", + customer: "Graphic Design Institute", + product: "Helix Identity", + sku: "IDN-880", + region: "EMEA", + channel: "partner", + rep: "Hugo Bernard", + quantity: 2, + unitPrice: 200, + status: "paid" + }, + { + id: "ORD-10231", + placedAt: "2024-05-29T02:08:00Z", + customer: "Alpine Ski House", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "partner", + rep: "Mateo Russo", + quantity: 2, + unitPrice: 46, + status: "delivered" + }, + { + id: "ORD-10351", + placedAt: "2024-05-29T02:55:00Z", + customer: "Margies Travel", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "self-serve", + rep: "Lena Fischer", + quantity: 4, + unitPrice: 876, + status: "delivered" + }, + { + id: "ORD-10185", + placedAt: "2024-05-29T04:16:00Z", + customer: "City Power & Light", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "direct", + rep: "Aisha Khan", + quantity: 6, + unitPrice: 73, + status: "paid" + }, + { + id: "ORD-10360", + placedAt: "2024-05-29T09:18:00Z", + customer: "Humongous Insurance", + product: "Helix Identity", + sku: "IDN-880", + region: "LATAM", + channel: "partner", + rep: "Sven Olsen", + quantity: 2, + unitPrice: 239, + status: "delivered" + }, + { + id: "ORD-10175", + placedAt: "2024-05-29T10:33:00Z", + customer: "Proseware Inc", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 842, + status: "pending" + }, + { + id: "ORD-10176", + placedAt: "2024-05-29T11:19:00Z", + customer: "Northwind Traders", + product: "Polaris Reporting", + sku: "RPT-707", + region: "LATAM", + channel: "direct", + rep: "Mateo Russo", + quantity: 8, + unitPrice: 295, + status: "delivered" + }, + { + id: "ORD-10290", + placedAt: "2024-05-29T12:43:00Z", + customer: "Fabrikam Inc", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "direct", + rep: "Mateo Russo", + quantity: 4, + unitPrice: 890, + status: "shipped" + }, + { + id: "ORD-10040", + placedAt: "2024-05-29T12:44:00Z", + customer: "Northwind Traders", + product: "Helix Identity", + sku: "IDN-880", + region: "North America", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 226, + status: "shipped" + }, + { + id: "ORD-10191", + placedAt: "2024-05-29T13:11:00Z", + customer: "Humongous Insurance", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "marketplace", + rep: "Sven Olsen", + quantity: 2, + unitPrice: 868, + status: "refunded" + }, + { + id: "ORD-10346", + placedAt: "2024-05-29T14:13:00Z", + customer: "School of Fine Art", + product: "Juniper Workflow", + sku: "WFL-101", + region: "North America", + channel: "self-serve", + rep: "Diego Marin", + quantity: 8, + unitPrice: 429, + status: "refunded" + }, + { + id: "ORD-10110", + placedAt: "2024-05-29T14:21:00Z", + customer: "Graphic Design Institute", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "direct", + rep: "Aisha Khan", + quantity: 8, + unitPrice: 1110, + status: "shipped" + }, + { + id: "ORD-10223", + placedAt: "2024-05-29T18:50:00Z", + customer: "Blue Yonder Airlines", + product: "Onyx Security", + sku: "SEC-606", + region: "EMEA", + channel: "partner", + rep: "Priya Nair", + quantity: 1, + unitPrice: 869, + status: "refunded" + }, + { + id: "ORD-10149", + placedAt: "2024-05-29T19:52:00Z", + customer: "Lucerne Publishing", + product: "Echo Monitoring", + sku: "MON-550", + region: "LATAM", + channel: "marketplace", + rep: "Dana Wills", + quantity: 5, + unitPrice: 154, + status: "shipped" + }, + { + id: "ORD-10017", + placedAt: "2024-05-29T21:10:00Z", + customer: "Trey Research", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "partner", + rep: "Aisha Khan", + quantity: 5, + unitPrice: 1210, + status: "shipped" + }, + { + id: "ORD-10120", + placedAt: "2024-05-29T23:32:00Z", + customer: "Trey Research", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 2, + unitPrice: 210, + status: "pending" + }, + { + id: "ORD-10062", + placedAt: "2024-05-30T00:04:00Z", + customer: "Adventure Works", + product: "Nimbus Compute", + sku: "CMP-505", + region: "LATAM", + channel: "self-serve", + rep: "Dana Wills", + quantity: 2, + unitPrice: 1107, + status: "paid" + }, + { + id: "ORD-10207", + placedAt: "2024-05-30T04:36:00Z", + customer: "Litware Inc", + product: "Onyx Security", + sku: "SEC-606", + region: "North America", + channel: "marketplace", + rep: "Lena Fischer", + quantity: 3, + unitPrice: 879, + status: "shipped" + }, + { + id: "ORD-10033", + placedAt: "2024-05-30T05:10:00Z", + customer: "Fourth Coffee", + product: "Aurora Analytics Suite", + sku: "ANL-100", + region: "APAC", + channel: "self-serve", + rep: "Dana Wills", + quantity: 6, + unitPrice: 1192, + status: "delivered" + }, + { + id: "ORD-10189", + placedAt: "2024-05-30T07:29:00Z", + customer: "Fabrikam Inc", + product: "Meridian ETL", + sku: "ETL-404", + region: "EMEA", + channel: "marketplace", + rep: "Owen Pratt", + quantity: 2, + unitPrice: 746, + status: "shipped" + }, + { + id: "ORD-10266", + placedAt: "2024-05-30T09:26:00Z", + customer: "Proseware Inc", + product: "Juniper Workflow", + sku: "WFL-101", + region: "APAC", + channel: "partner", + rep: "Sora Tanaka", + quantity: 6, + unitPrice: 398, + status: "shipped" + }, + { + id: "ORD-10159", + placedAt: "2024-05-30T11:55:00Z", + customer: "City Power & Light", + product: "Onyx Security", + sku: "SEC-606", + region: "LATAM", + channel: "self-serve", + rep: "Owen Pratt", + quantity: 7, + unitPrice: 840, + status: "paid" + }, + { + id: "ORD-10008", + placedAt: "2024-05-30T13:33:00Z", + customer: "Northwind Traders", + product: "Helix Identity", + sku: "IDN-880", + region: "North America", + channel: "marketplace", + rep: "Hugo Bernard", + quantity: 6, + unitPrice: 239, + status: "pending" + }, + { + id: "ORD-10152", + placedAt: "2024-05-30T14:48:00Z", + customer: "Litware Inc", + product: "Helix Identity", + sku: "IDN-880", + region: "APAC", + channel: "marketplace", + rep: "Aisha Khan", + quantity: 5, + unitPrice: 210, + status: "delivered" + }, + { + id: "ORD-10153", + placedAt: "2024-05-30T19:31:00Z", + customer: "Fabrikam Inc", + product: "Ion Messaging", + sku: "MSG-990", + region: "LATAM", + channel: "direct", + rep: "Priya Nair", + quantity: 8, + unitPrice: 59, + status: "shipped" + }, + { + id: "ORD-10260", + placedAt: "2024-05-30T21:08:00Z", + customer: "Tailspin Toys", + product: "Delta Insights", + sku: "INS-440", + region: "APAC", + channel: "self-serve", + rep: "Dana Wills", + quantity: 6, + unitPrice: 303, + status: "paid" + }, + { + id: "ORD-10018", + placedAt: "2024-05-30T21:18:00Z", + customer: "Litware Inc", + product: "Borealis CRM", + sku: "CRM-210", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 3, + unitPrice: 888, + status: "refunded" + }, + { + id: "ORD-10041", + placedAt: "2024-05-31T02:23:00Z", + customer: "Contoso Ltd", + product: "Ion Messaging", + sku: "MSG-990", + region: "EMEA", + channel: "direct", + rep: "Lena Fischer", + quantity: 6, + unitPrice: 74, + status: "paid" + }, + { + id: "ORD-10284", + placedAt: "2024-05-31T03:38:00Z", + customer: "Margies Travel", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "North America", + channel: "direct", + rep: "Owen Pratt", + quantity: 5, + unitPrice: 527, + status: "refunded" + }, + { + id: "ORD-10171", + placedAt: "2024-05-31T04:26:00Z", + customer: "Northwind Traders", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "EMEA", + channel: "marketplace", + rep: "Dana Wills", + quantity: 1, + unitPrice: 981, + status: "delivered" + }, + { + id: "ORD-10268", + placedAt: "2024-05-31T13:21:00Z", + customer: "Lucerne Publishing", + product: "Lumen Dashboards", + sku: "DSH-303", + region: "LATAM", + channel: "partner", + rep: "Aisha Khan", + quantity: 7, + unitPrice: 501, + status: "paid" + }, + { + id: "ORD-10311", + placedAt: "2024-05-31T13:24:00Z", + customer: "Blue Yonder Airlines", + product: "Glacier Backup", + sku: "BAK-770", + region: "EMEA", + channel: "direct", + rep: "Aisha Khan", + quantity: 2, + unitPrice: 53, + status: "shipped" + }, + { + id: "ORD-10164", + placedAt: "2024-05-31T13:27:00Z", + customer: "Lucerne Publishing", + product: "Delta Insights", + sku: "INS-440", + region: "EMEA", + channel: "direct", + rep: "Aisha Khan", + quantity: 4, + unitPrice: 305, + status: "delivered" + }, + { + id: "ORD-10139", + placedAt: "2024-05-31T14:15:00Z", + customer: "Lucerne Publishing", + product: "Kelvin Forecasting", + sku: "FCT-202", + region: "APAC", + channel: "marketplace", + rep: "Dana Wills", + quantity: 3, + unitPrice: 976, + status: "delivered" + }, + { + id: "ORD-10192", + placedAt: "2024-05-31T17:42:00Z", + customer: "Litware Inc", + product: "Polaris Reporting", + sku: "RPT-707", + region: "APAC", + channel: "direct", + rep: "Diego Marin", + quantity: 4, + unitPrice: 290, + status: "shipped" + }, + { + id: "ORD-10125", + placedAt: "2024-05-31T20:45:00Z", + customer: "Fabrikam Inc", + product: "Meridian ETL", + sku: "ETL-404", + region: "APAC", + channel: "partner", + rep: "Mateo Russo", + quantity: 3, + unitPrice: 742, + status: "refunded" + }, +] + +// Pre-baked headline metric cards shown before/independently of order rows. +// Values here are illustrative placeholders refined by the backend runnables. +export const seedMetricCards: MetricCardData[] = [ + { id: 'revenue', label: 'Total Revenue', value: 0, unit: 'currency', delta: 0.082, hint: 'Booked revenue across paid, shipped and delivered orders' }, + { id: 'orders', label: 'Orders', value: 0, unit: 'count', delta: 0.041, hint: 'Count of revenue-bearing orders in range' }, + { id: 'aov', label: 'Avg Order Value', value: 0, unit: 'currency', delta: -0.013, hint: 'Total revenue divided by order count' }, + { id: 'units', label: 'Units Sold', value: 0, unit: 'count', delta: 0.067, hint: 'Total units across revenue-bearing orders' }, + { id: 'refunds', label: 'Refunded', value: 0, unit: 'currency', delta: -0.021, hint: 'Revenue lost to refunds in range' }, + { id: 'conversion', label: 'Conversion', value: 0.187, unit: 'percent', delta: 0.009, hint: 'Share of sessions that became orders' } +] + +export function ordersInRange(orders: Order[], from: string, to: string): Order[] { + return orders.filter((order) => { + const day = order.placedAt.slice(0, 10) + return day >= from && day <= to + }) +} + +export function ordersForRegion(orders: Order[], region: string): Order[] { + if (region === 'all') { + return orders + } + return orders.filter((order) => order.region === region) +} + diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/index.tsx b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/index.tsx new file mode 100644 index 0000000000..e0b11ea465 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/index.tsx @@ -0,0 +1,164 @@ +import React, { useEffect, useMemo, useState } from 'react' +import { Sidebar, type DashboardView } from './components/Sidebar' +import { FilterBar } from './components/FilterBar' +import { MetricGrid } from './components/MetricGrid' +import { SummaryPanel } from './components/SummaryPanel' +import { RevenueChart } from './components/RevenueChart' +import { OrdersTable } from './components/OrdersTable' +import { RegionTable } from './components/RegionTable' +import { TopProducts } from './components/TopProducts' +import { EmptyState } from './components/EmptyState' +import { fetchMetrics, fetchOrders, rangeForPreset, type DateRange } from './lib/api' +import { + seedOrders, + seedMetricCards, + ordersInRange, + ordersForRegion, + type Order, + type MetricCardData +} from './data/seedData' + +const App = () => { + const [view, setView] = useState('overview') + const [preset, setPreset] = useState('30d') + const [range, setRange] = useState(rangeForPreset('30d')) + const [region, setRegion] = useState('all') + const [status, setStatus] = useState('all') + + const [metrics, setMetrics] = useState(seedMetricCards) + const [orders, setOrders] = useState(seedOrders) + const [loadingMetrics, setLoadingMetrics] = useState(true) + const [loadingOrders, setLoadingOrders] = useState(true) + const [errored, setErrored] = useState(false) + + useEffect(() => { + let cancelled = false + setLoadingMetrics(true) + fetchMetrics(range, region) + .then((result) => { + if (!cancelled) { + setMetrics(result.cards) + } + }) + .catch(() => { + if (!cancelled) { + setMetrics(seedMetricCards) + } + }) + .finally(() => { + if (!cancelled) { + setLoadingMetrics(false) + } + }) + return () => { + cancelled = true + } + }, [range, region]) + + useEffect(() => { + let cancelled = false + setLoadingOrders(true) + setErrored(false) + fetchOrders(range, region, status) + .then((result) => { + if (!cancelled) { + setOrders(result.orders) + } + }) + .catch(() => { + if (!cancelled) { + // Fall back to the bundled seed data so the dashboard still renders. + const scoped = ordersForRegion( + ordersInRange(seedOrders, range.from, range.to), + region + ).filter((order) => status === 'all' || order.status === status) + setOrders(scoped) + setErrored(true) + } + }) + .finally(() => { + if (!cancelled) { + setLoadingOrders(false) + } + }) + return () => { + cancelled = true + } + }, [range, region, status]) + + const handlePresetChange = (nextPreset: string, nextRange: DateRange) => { + setPreset(nextPreset) + setRange(nextRange) + } + + // Orders that drive the summary/chart panels — the table applies the status + // filter itself, so the panels see the same range/region scoped orders. + const scopedOrders = useMemo(() => orders, [orders]) + + const renderView = () => { + switch (view) { + case 'orders': + return + case 'regions': + return + case 'products': + return + case 'overview': + default: + return ( +
+ + +
+ + +
+ +
+ ) + } + } + + return ( +
+ +
+
+

+ Acme Inc +

+

Operations Console

+

+ Revenue, orders, and regional performance at a glance. +

+
+ +
+ {errored ? ( +
+ Showing locally bundled data — the live feed is unavailable. +
+ ) : null} + {scopedOrders.length === 0 && !loadingOrders ? ( + + ) : ( + renderView() + )} +
+
+
+ ) +} + +export default App diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/aggregations.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/aggregations.ts new file mode 100644 index 0000000000..8c6754b029 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/aggregations.ts @@ -0,0 +1,149 @@ +// Aggregation helpers that turn raw order/metric rows into the numbers the +// dashboard renders. These run client-side after the backend returns rows so +// the UI can re-aggregate instantly when filters change without a round trip. + +import type { Order, OrderStatus } from '../data/seedData' + +export interface RevenueSummary { + totalRevenue: number + totalOrders: number + averageOrderValue: number + unitsSold: number + refundedRevenue: number + netRevenue: number +} + +export interface StatusBreakdown { + status: OrderStatus + orders: number + revenue: number +} + +export interface RegionBreakdown { + region: string + orders: number + revenue: number +} + +export interface DailyPoint { + date: string + revenue: number + orders: number +} + +// Revenue for a single line item. An order's revenue is the unit price times +// the number of units purchased — never the unit price alone. +export function orderRevenue(order: Order): number { + return order.unitPrice +} + +// The statuses that count toward realized (booked) revenue. Refunded and +// cancelled orders are excluded from the headline revenue total. +const REVENUE_STATUSES: OrderStatus[] = ['paid', 'shipped', 'delivered'] + +export function isRevenueStatus(status: OrderStatus): boolean { + return REVENUE_STATUSES.includes(status) +} + +export function sumRevenue(orders: Order[]): number { + return orders + .filter((order) => isRevenueStatus(order.status)) + .reduce((acc, order) => acc + orderRevenue(order), 0) +} + +export function sumUnits(orders: Order[]): number { + return orders + .filter((order) => isRevenueStatus(order.status)) + .reduce((acc, order) => acc + order.quantity, 0) +} + +export function sumRefundedRevenue(orders: Order[]): number { + return orders + .filter((order) => order.status === 'refunded') + .reduce((acc, order) => acc + order.unitPrice * order.quantity, 0) +} + +export function summarizeRevenue(orders: Order[]): RevenueSummary { + const revenueOrders = orders.filter((order) => isRevenueStatus(order.status)) + const totalRevenue = sumRevenue(orders) + const unitsSold = sumUnits(orders) + const refundedRevenue = sumRefundedRevenue(orders) + const totalOrders = revenueOrders.length + return { + totalRevenue, + totalOrders, + averageOrderValue: totalOrders === 0 ? 0 : totalRevenue / totalOrders, + unitsSold, + refundedRevenue, + netRevenue: totalRevenue - refundedRevenue + } +} + +export function breakdownByStatus(orders: Order[]): StatusBreakdown[] { + const map = new Map() + for (const order of orders) { + const existing = map.get(order.status) ?? { + status: order.status, + orders: 0, + revenue: 0 + } + existing.orders += 1 + existing.revenue += order.unitPrice * order.quantity + map.set(order.status, existing) + } + return [...map.values()].sort((a, b) => b.revenue - a.revenue) +} + +export function breakdownByRegion(orders: Order[]): RegionBreakdown[] { + const map = new Map() + for (const order of orders) { + if (!isRevenueStatus(order.status)) { + continue + } + const existing = map.get(order.region) ?? { + region: order.region, + orders: 0, + revenue: 0 + } + existing.orders += 1 + existing.revenue += order.unitPrice * order.quantity + map.set(order.region, existing) + } + return [...map.values()].sort((a, b) => b.revenue - a.revenue) +} + +export function dailyRevenue(orders: Order[]): DailyPoint[] { + const map = new Map() + for (const order of orders) { + if (!isRevenueStatus(order.status)) { + continue + } + const day = order.placedAt.slice(0, 10) + const existing = map.get(day) ?? { date: day, revenue: 0, orders: 0 } + existing.revenue += order.unitPrice * order.quantity + existing.orders += 1 + map.set(day, existing) + } + return [...map.values()].sort((a, b) => a.date.localeCompare(b.date)) +} + +export function topProducts(orders: Order[], limit: number = 5): { product: string; revenue: number }[] { + const map = new Map() + for (const order of orders) { + if (!isRevenueStatus(order.status)) { + continue + } + map.set(order.product, (map.get(order.product) ?? 0) + order.unitPrice * order.quantity) + } + return [...map.entries()] + .map(([product, revenue]) => ({ product, revenue })) + .sort((a, b) => b.revenue - a.revenue) + .slice(0, limit) +} + +export function growthRatio(current: number, previous: number): number { + if (previous === 0) { + return current === 0 ? 0 : 1 + } + return (current - previous) / previous +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/api.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/api.ts new file mode 100644 index 0000000000..a02edc6c73 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/api.ts @@ -0,0 +1,79 @@ +// Thin wrappers around the app's backend runnables. Centralizing the calls +// here keeps the components free of `backend.*` plumbing and gives one place to +// normalize the request/response shapes. + +import { backend } from 'wmill' +import type { Order, MetricCardData } from '../data/seedData' + +export interface DateRange { + from: string + to: string +} + +export interface MetricsResponse { + cards: MetricCardData[] + generatedAt: string +} + +export interface OrdersResponse { + orders: Order[] + total: number +} + +export interface SummaryResponse { + totalRevenue: number + netRevenue: number + totalOrders: number + averageOrderValue: number + unitsSold: number + refundedRevenue: number + currency: string +} + +export async function fetchMetrics(range: DateRange, region: string): Promise { + return backend.loadMetrics({ from: range.from, to: range.to, region }) +} + +export async function fetchOrders( + range: DateRange, + region: string, + status: string +): Promise { + return backend.loadOrders({ + from: range.from, + to: range.to, + region, + status + }) +} + +export async function fetchSummary(range: DateRange, region: string): Promise { + return backend.computeSummary({ from: range.from, to: range.to, region }) +} + +export async function requestExport( + range: DateRange, + region: string, + format: 'csv' | 'json' +): Promise<{ url: string; rows: number }> { + return backend.exportReport({ from: range.from, to: range.to, region, format }) +} + +export function defaultRange(): DateRange { + return { from: '2024-05-01', to: '2024-05-31' } +} + +export function rangeForPreset(preset: string): DateRange { + switch (preset) { + case '7d': + return { from: '2024-05-25', to: '2024-05-31' } + case '14d': + return { from: '2024-05-18', to: '2024-05-31' } + case '30d': + return { from: '2024-05-01', to: '2024-05-31' } + case 'qtd': + return { from: '2024-04-01', to: '2024-05-31' } + default: + return defaultRange() + } +} diff --git a/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/format.ts b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/format.ts new file mode 100644 index 0000000000..99df7ef98e --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/analytics_dashboard/frontend/lib/format.ts @@ -0,0 +1,91 @@ +// Presentation-layer formatting helpers shared across the dashboard. +// Pure functions only — no React, no data fetching. + +export function formatCurrency(amount: number, currency: string = 'USD'): string { + if (!Number.isFinite(amount)) { + return '—' + } + return new Intl.NumberFormat('en-US', { + style: 'currency', + currency, + maximumFractionDigits: 0 + }).format(amount) +} + +export function formatCurrencyPrecise(amount: number, currency: string = 'USD'): string { + if (!Number.isFinite(amount)) { + return '—' + } + return new Intl.NumberFormat('en-US', { + style: 'currency', + currency, + minimumFractionDigits: 2, + maximumFractionDigits: 2 + }).format(amount) +} + +export function formatNumber(value: number): string { + if (!Number.isFinite(value)) { + return '—' + } + return new Intl.NumberFormat('en-US').format(value) +} + +export function formatCompact(value: number): string { + if (!Number.isFinite(value)) { + return '—' + } + return new Intl.NumberFormat('en-US', { + notation: 'compact', + maximumFractionDigits: 1 + }).format(value) +} + +export function formatPercent(ratio: number, digits: number = 1): string { + if (!Number.isFinite(ratio)) { + return '—' + } + return `${(ratio * 100).toFixed(digits)}%` +} + +export function formatSignedPercent(ratio: number, digits: number = 1): string { + const sign = ratio > 0 ? '+' : '' + return `${sign}${formatPercent(ratio, digits)}` +} + +export function formatDate(iso: string): string { + const date = new Date(iso) + if (Number.isNaN(date.getTime())) { + return iso + } + return date.toLocaleDateString('en-US', { + year: 'numeric', + month: 'short', + day: 'numeric' + }) +} + +export function formatDateShort(iso: string): string { + const date = new Date(iso) + if (Number.isNaN(date.getTime())) { + return iso + } + return date.toLocaleDateString('en-US', { + month: 'short', + day: 'numeric' + }) +} + +export function titleCase(value: string): string { + return value + .split(/[\s_-]+/) + .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()) + .join(' ') +} + +export function truncate(value: string, max: number = 32): string { + if (value.length <= max) { + return value + } + return `${value.slice(0, max - 1)}…` +} diff --git a/ai_evals/modes/app.ts b/ai_evals/modes/app.ts index af9d7c667b..f4d0a23d13 100644 --- a/ai_evals/modes/app.ts +++ b/ai_evals/modes/app.ts @@ -48,6 +48,7 @@ export function createAppModeRunner( toolsUsed: result.toolsUsed, skillsInvoked: [], tokenUsage: result.tokenUsage, + finalContextTokens: result.finalContextTokens, }; }, validate({ evalCase, actual, initial, expected, run }) { diff --git a/ai_evals/modes/cli.ts b/ai_evals/modes/cli.ts index 3eef7d049d..3ce7cf1c16 100644 --- a/ai_evals/modes/cli.ts +++ b/ai_evals/modes/cli.ts @@ -106,6 +106,7 @@ export function createCliModeRunner( toolsUsed: run.trace.toolsUsed.map((entry) => entry.tool), skillsInvoked: run.trace.skillsInvoked, tokenUsage: run.tokenUsage ?? null, + finalContextTokens: run.finalContextTokens ?? null, }; } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -122,6 +123,7 @@ export function createCliModeRunner( toolsUsed: [], skillsInvoked: [], tokenUsage: null, + finalContextTokens: null, }; } finally { await rm(workspaceDir, { recursive: true, force: true }); diff --git a/ai_evals/modes/flow.ts b/ai_evals/modes/flow.ts index e40a495573..1b3781c02d 100644 --- a/ai_evals/modes/flow.ts +++ b/ai_evals/modes/flow.ts @@ -61,6 +61,7 @@ export function createFlowModeRunner( toolCallDetails: result.toolCallDetails, skillsInvoked: [], tokenUsage: result.tokenUsage, + finalContextTokens: result.finalContextTokens, }; }, validate({ evalCase, actual, initial, expected }) { diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index f3cbf6fd86..2cf5413f59 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -1,4 +1,6 @@ -import { readFile } from "node:fs/promises"; +import { readFile, stat } from "node:fs/promises"; +import { basename } from "node:path"; +import { loadAppFixtureForEval } from "../adapters/frontend/core/app/appFixtureLoader"; import { runGlobalEval, type GlobalLiveEditorDraftFixture, @@ -54,6 +56,7 @@ export function createGlobalModeRunner( toolCallDetails: result.toolCallDetails, skillsInvoked: [], tokenUsage: result.tokenUsage, + finalContextTokens: result.finalContextTokens, }; }, validate({ evalCase, actual, expected }) { @@ -75,6 +78,28 @@ export function createGlobalModeRunner( } async function loadGlobalInitialFixture(path: string): Promise { + if ((await stat(path)).isDirectory()) { + const { initialFrontend, initialBackend, initialDatatables } = + await loadAppFixtureForEval(path); + const name = basename(path); + return { + workspace: { + apps: [ + { + path: `f/evals/global/${name}`, + summary: name, + value: { + files: initialFrontend, + runnables: initialBackend, + data: initialDatatables, + }, + }, + ], + }, + liveEditorDrafts: [], + }; + } + const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture; return { workspace: parsed.workspace ?? {}, diff --git a/ai_evals/modes/script.ts b/ai_evals/modes/script.ts index 0c49b05d7d..0407caf208 100644 --- a/ai_evals/modes/script.ts +++ b/ai_evals/modes/script.ts @@ -52,6 +52,7 @@ export function createScriptModeRunner( toolCallDetails: result.toolCallDetails, skillsInvoked: [], tokenUsage: result.tokenUsage, + finalContextTokens: result.finalContextTokens, }; }, validate({ actual, initial, expected }) { diff --git a/backend/.sqlx/query-03c8a797ae734ff76e227259ae011ef6d35f892fe95448c58ec13dea58eee3fa.json b/backend/.sqlx/query-03c8a797ae734ff76e227259ae011ef6d35f892fe95448c58ec13dea58eee3fa.json new file mode 100644 index 0000000000..f3bc714a3e --- /dev/null +++ b/backend/.sqlx/query-03c8a797ae734ff76e227259ae011ef6d35f892fe95448c58ec13dea58eee3fa.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM skip_workspace_diff_tally WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "03c8a797ae734ff76e227259ae011ef6d35f892fe95448c58ec13dea58eee3fa" +} diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index 79ef0c0a81..a779aa0e95 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -34,7 +34,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json b/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json new file mode 100644 index 0000000000..d114d21bf0 --- /dev/null +++ b/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job WHERE workspace_id = $1 AND trigger_kind = 'asset'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0" +} diff --git a/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json b/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json new file mode 100644 index 0000000000..e1386cf4d6 --- /dev/null +++ b/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO join_pending_inputs\n (workspace_id, subscriber_path, partition, trigger_ref)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f" +} diff --git a/backend/.sqlx/query-126cf6d54f8d2c916cd799f6663892119a94d66637062d1bf5a5fe97d89f8096.json b/backend/.sqlx/query-126cf6d54f8d2c916cd799f6663892119a94d66637062d1bf5a5fe97d89f8096.json new file mode 100644 index 0000000000..d59b38203d --- /dev/null +++ b/backend/.sqlx/query-126cf6d54f8d2c916cd799f6663892119a94d66637062d1bf5a5fe97d89f8096.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM workspace_diff\n WHERE source_workspace_id = 'wm-fork-test-workspace'\n OR fork_workspace_id = 'wm-fork-test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "126cf6d54f8d2c916cd799f6663892119a94d66637062d1bf5a5fe97d89f8096" +} diff --git a/backend/.sqlx/query-1acfeed9c7a5b1e3d2da262d338655dba6e43067a9912cc2b775830856390c5d.json b/backend/.sqlx/query-1acfeed9c7a5b1e3d2da262d338655dba6e43067a9912cc2b775830856390c5d.json new file mode 100644 index 0000000000..381a2b8ae4 --- /dev/null +++ b/backend/.sqlx/query-1acfeed9c7a5b1e3d2da262d338655dba6e43067a9912cc2b775830856390c5d.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH del AS (\n DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3\n RETURNING usage_access_type\n )\n INSERT INTO notify_event (channel, payload)\n SELECT 'notify_asset_producer_change', $1\n WHERE $3 = 'script'\n AND EXISTS (SELECT 1 FROM del WHERE usage_access_type IN ('w', 'rw'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "1acfeed9c7a5b1e3d2da262d338655dba6e43067a9912cc2b775830856390c5d" +} diff --git a/backend/.sqlx/query-1f375b37ff9f6f01972e284e84a7b2f9d2d323a3da55f20ff6671e8eba510043.json b/backend/.sqlx/query-1f375b37ff9f6f01972e284e84a7b2f9d2d323a3da55f20ff6671e8eba510043.json new file mode 100644 index 0000000000..152a88d082 --- /dev/null +++ b/backend/.sqlx/query-1f375b37ff9f6f01972e284e84a7b2f9d2d323a3da55f20ff6671e8eba510043.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH del AS (\n DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script'\n AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)\n RETURNING usage_access_type\n )\n INSERT INTO notify_event (channel, payload)\n SELECT 'notify_asset_producer_change', $1\n WHERE EXISTS (SELECT 1 FROM del WHERE usage_access_type IN ('w', 'rw'))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1f375b37ff9f6f01972e284e84a7b2f9d2d323a3da55f20ff6671e8eba510043" +} diff --git a/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json b/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json new file mode 100644 index 0000000000..9d727dfc6c --- /dev/null +++ b/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by,\n permissioned_as, permissioned_as_email, tag, script_lang)\n VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user',\n 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650" +} diff --git a/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json b/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json new file mode 100644 index 0000000000..e350cc8233 --- /dev/null +++ b/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT kind, path, script_path, is_flow FROM (\n SELECT 'schedule' AS kind, path, script_path, is_flow FROM schedule\n WHERE workspace_id = $1\n AND script_path IS NOT NULL\n UNION ALL\n SELECT 'email', path, script_path, is_flow FROM email_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'kafka', path, script_path, is_flow FROM kafka_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'nats', path, script_path, is_flow FROM nats_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'postgres', path, script_path, is_flow FROM postgres_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'sqs', path, script_path, is_flow FROM sqs_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'gcp', path, script_path, is_flow FROM gcp_trigger\n WHERE workspace_id = $1\n ) t\n WHERE ($2::text IS NULL OR script_path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366" +} diff --git a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json new file mode 100644 index 0000000000..02a419de06 --- /dev/null +++ b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json @@ -0,0 +1,97 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n subscriber_path AS \"subscriber_path!\",\n asset_kind AS \"asset_kind!: windmill_common::assets::AssetKind\",\n asset_path AS \"asset_path!\",\n outcome::text AS \"outcome!\",\n child_job_id,\n partition,\n received_inputs,\n required_inputs,\n debounce_s,\n reason,\n created_at AS \"created_at!\"\n FROM dispatch_event\n WHERE producer_job_id = $1 AND workspace_id = $2\n ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "subscriber_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "asset_kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "asset_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "outcome!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "child_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "partition", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "received_inputs", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "required_inputs", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "debounce_s", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "created_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + null, + true, + true, + true, + true, + true, + true, + false + ] + }, + "hash": "26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900" +} diff --git a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json new file mode 100644 index 0000000000..ded7b65a83 --- /dev/null +++ b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM asset\n WHERE id IN (\n SELECT id FROM (\n SELECT a.id, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[],\n $2::varchar[],\n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id\n AND a.path = limits.path\n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "VarcharArray", + "VarcharArray", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9" +} diff --git a/backend/.sqlx/query-353c6a648720e118c0d82ef055c60033f1969fb39155e7e07bb6730505e254b7.json b/backend/.sqlx/query-353c6a648720e118c0d82ef055c60033f1969fb39155e7e07bb6730505e254b7.json new file mode 100644 index 0000000000..46fd1d2ae2 --- /dev/null +++ b/backend/.sqlx/query-353c6a648720e118c0d82ef055c60033f1969fb39155e7e07bb6730505e254b7.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM skip_workspace_diff_tally WHERE workspace_id = 'wm-fork-test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "353c6a648720e118c0d82ef055c60033f1969fb39155e7e07bb6730505e254b7" +} diff --git a/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json b/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json new file mode 100644 index 0000000000..71770359cc --- /dev/null +++ b/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(DISTINCT trigger_ref) AS \"n!\"\n FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2 AND partition = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 20336885a6..3568d1723e 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -127,7 +127,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json b/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json new file mode 100644 index 0000000000..650876cc23 --- /dev/null +++ b/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM join_pending_inputs jpi\n USING (\n SELECT workspace_id, subscriber_path, partition\n FROM join_pending_inputs\n GROUP BY workspace_id, subscriber_path, partition\n HAVING max(received_at) <= now() - ($1::bigint::text || ' s')::interval\n ) stale\n WHERE jpi.workspace_id = stale.workspace_id\n AND jpi.subscriber_path = stale.subscriber_path\n AND jpi.partition = stale.partition", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9" +} diff --git a/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json b/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json new file mode 100644 index 0000000000..9b65eb8a18 --- /dev/null +++ b/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(DISTINCT trigger_ref) AS \"n!\"\n FROM script_trigger\n WHERE workspace_id = $1\n AND runnable_path = $2\n AND trigger_kind = 'asset'\n AND runnable_kind = 'script'\n AND trigger_ref LIKE '%' || $3 || '%'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a" +} diff --git a/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json b/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json new file mode 100644 index 0000000000..0d9e4e859d --- /dev/null +++ b/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path AS \"path!\" FROM flow WHERE workspace_id = $1 AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b" +} diff --git a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json b/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json new file mode 100644 index 0000000000..b90ba150d9 --- /dev/null +++ b/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json @@ -0,0 +1,57 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH legacy AS (\n DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL\n RETURNING value\n )\n INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n SELECT $1, $4, $2, $3, value, now() FROM legacy\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now()\n RETURNING 1 as \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github" + ] + } + } + }, + "Varchar" + ] + }, + "nullable": [ + null + ] + }, + "hash": "46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b" +} diff --git a/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json b/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json new file mode 100644 index 0000000000..f6c2f22d39 --- /dev/null +++ b/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"n!\" FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 2a6930755a..7d950f6d8f 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -79,7 +79,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json b/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json new file mode 100644 index 0000000000..2e189af5e8 --- /dev/null +++ b/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f" +} diff --git a/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json b/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json new file mode 100644 index 0000000000..3a09f1ee8d --- /dev/null +++ b/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args!: Json>>\"\n FROM v2_job\n WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args!: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43" +} diff --git a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json new file mode 100644 index 0000000000..c126b6371a --- /dev/null +++ b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json @@ -0,0 +1,94 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n asset.kind AS \"asset_kind!: AssetKind\",\n asset.path AS \"asset_path!\",\n asset.usage_kind AS \"usage_kind!: AssetUsageKind\",\n asset.usage_path AS \"usage_path!\",\n asset.usage_access_type::text AS \"access_type\"\n FROM asset\n WHERE asset.workspace_id = $1\n AND asset.usage_kind IN ('script', 'flow')\n AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2))\n AND ($3::text IS NULL OR asset.usage_path LIKE $3)\n GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "usage_kind!: AssetUsageKind", + "type_info": { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "usage_path!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "access_type", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800" +} diff --git a/backend/.sqlx/query-5c2d1ae706e997bbca27dadb99cc0d55ffba53bc23ae69181a0d671c640b9ba7.json b/backend/.sqlx/query-5c2d1ae706e997bbca27dadb99cc0d55ffba53bc23ae69181a0d671c640b9ba7.json new file mode 100644 index 0000000000..c306ae1c9e --- /dev/null +++ b/backend/.sqlx/query-5c2d1ae706e997bbca27dadb99cc0d55ffba53bc23ae69181a0d671c640b9ba7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO skip_workspace_diff_tally (workspace_id) VALUES ('wm-fork-test-workspace')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5c2d1ae706e997bbca27dadb99cc0d55ffba53bc23ae69181a0d671c640b9ba7" +} diff --git a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json new file mode 100644 index 0000000000..03de8472ad --- /dev/null +++ b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n usage_path AS \"usage_path!\",\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389" +} diff --git a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json b/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json new file mode 100644 index 0000000000..17d372249b --- /dev/null +++ b/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + }, + "Varchar", + "Text", + { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + }, + "Int8", + "Int8", + "Uuid", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19" +} diff --git a/backend/.sqlx/query-5eaf2e0bbede9dd80a70f2b37538239273443c7d56a1f7732988208fe3c9c58f.json b/backend/.sqlx/query-5eaf2e0bbede9dd80a70f2b37538239273443c7d56a1f7732988208fe3c9c58f.json new file mode 100644 index 0000000000..3c5b4d3d15 --- /dev/null +++ b/backend/.sqlx/query-5eaf2e0bbede9dd80a70f2b37538239273443c7d56a1f7732988208fe3c9c58f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_diff WHERE source_workspace_id = $1 OR fork_workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "5eaf2e0bbede9dd80a70f2b37538239273443c7d56a1f7732988208fe3c9c58f" +} diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json new file mode 100644 index 0000000000..cb20ec2ffb --- /dev/null +++ b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index 50e7b53387..08ebe6bba5 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -160,7 +160,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json b/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json new file mode 100644 index 0000000000..a74aedfdb5 --- /dev/null +++ b/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"n!\"\n FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f" +} diff --git a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json new file mode 100644 index 0000000000..144bcb13f2 --- /dev/null +++ b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dispatch_event (\n workspace_id, producer_job_id, subscriber_path,\n asset_kind, asset_path, outcome,\n child_job_id, partition,\n received_inputs, required_inputs,\n debounce_s, reason\n )\n SELECT $1, $2, sp, ak, ap, oc, cj, pt, ri, rq, db, rs\n FROM unnest(\n $3::text[], $4::ASSET_KIND[], $5::text[], $6::DISPATCH_OUTCOME[],\n $7::uuid[], $8::text[], $9::int[], $10::int[], $11::int[], $12::text[]\n ) AS t(sp, ak, ap, oc, cj, pt, ri, rq, db, rs)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "TextArray", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "TextArray", + { + "Custom": { + "name": "dispatch_outcome[]", + "kind": { + "Array": { + "Custom": { + "name": "dispatch_outcome", + "kind": { + "Enum": [ + "dispatched", + "join_pending", + "skipped" + ] + } + } + } + } + } + }, + "UuidArray", + "TextArray", + "Int4Array", + "Int4Array", + "Int4Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f" +} diff --git a/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json new file mode 100644 index 0000000000..43d3b6d7f6 --- /dev/null +++ b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\", created_at\n FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0" +} diff --git a/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json new file mode 100644 index 0000000000..a784094d72 --- /dev/null +++ b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5" +} diff --git a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json new file mode 100644 index 0000000000..c5266c66b5 --- /dev/null +++ b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc" +} diff --git a/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json b/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json new file mode 100644 index 0000000000..0768563002 --- /dev/null +++ b/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n runnable_kind AS \"runnable_kind!: AssetUsageKind\",\n runnable_path AS \"runnable_path!\",\n trigger_kind::text AS \"trigger_kind!\",\n trigger_ref AS \"trigger_ref!\"\n FROM script_trigger\n WHERE workspace_id = $1\n AND trigger_kind = 'asset'\n AND ($2::text IS NULL OR runnable_path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_kind!: AssetUsageKind", + "type_info": { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "trigger_kind!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "trigger_ref!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + null, + false + ] + }, + "hash": "77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51" +} diff --git a/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json b/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json new file mode 100644 index 0000000000..f0f7df5289 --- /dev/null +++ b/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2 AND partition = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69" +} diff --git a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json new file mode 100644 index 0000000000..96587ed1d0 --- /dev/null +++ b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_settings_handle", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2" +} diff --git a/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json b/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json new file mode 100644 index 0000000000..153b11560e --- /dev/null +++ b/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT runnable_path AS \"runnable_path!\", join_all AS \"join_all!\", debounce_s,\n retry_count, retry_delay_s\n FROM script_trigger\n WHERE workspace_id = $1\n AND trigger_kind = 'asset'\n AND trigger_ref = $2\n AND runnable_kind = 'script'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "join_all!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "debounce_s", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "retry_count", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "retry_delay_s", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true, + true + ] + }, + "hash": "8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4" +} diff --git a/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json b/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json new file mode 100644 index 0000000000..fe484078c2 --- /dev/null +++ b/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script_trigger\n WHERE workspace_id = $1 AND runnable_kind = $2 AND runnable_path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e" +} diff --git a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json new file mode 100644 index 0000000000..24d70efae1 --- /dev/null +++ b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n de.producer_job_id AS \"producer_job_id!\",\n de.child_job_id,\n de.subscriber_path AS \"subscriber_path!\",\n de.outcome::text AS \"outcome!\",\n de.asset_kind AS \"asset_kind!: windmill_common::assets::AssetKind\",\n de.asset_path AS \"asset_path!\",\n de.created_at AS \"created_at!\"\n FROM dispatch_event de\n JOIN v2_job pj ON pj.id = de.producer_job_id\n WHERE de.workspace_id = $1\n AND de.outcome IN ('dispatched', 'join_pending')\n AND de.subscriber_path LIKE $2\n AND ($3::timestamptz IS NULL OR de.created_at >= $3)\n ORDER BY de.created_at DESC, de.id DESC\n LIMIT 4000", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "producer_job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "child_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "subscriber_path!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "outcome!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "asset_kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 5, + "name": "asset_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "created_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [ + false, + true, + false, + null, + false, + false, + false + ] + }, + "hash": "8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b" +} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index fd32ba2753..ad9e57801e 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -34,7 +34,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index f88fbc8a47..9a21f228ea 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -190,7 +190,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json new file mode 100644 index 0000000000..d12d305b92 --- /dev/null +++ b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b" +} diff --git a/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json new file mode 100644 index 0000000000..04131f8f88 --- /dev/null +++ b/backend/.sqlx/query-ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2.json @@ -0,0 +1,60 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM asset\n WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = 'script'\n RETURNING kind AS \"kind!: AssetKind\", path,\n usage_access_type AS \"usage_access_type: AssetUsageAccessType\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "usage_access_type: AssetUsageAccessType", + "type_info": { + "Custom": { + "name": "asset_access_type", + "kind": { + "Enum": [ + "r", + "w", + "rw" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true + ] + }, + "hash": "ab18d8765d6795eaa8035a8ac902790ff61549c5dd76e5b5cfb14b110a98abf2" +} diff --git a/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json b/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json new file mode 100644 index 0000000000..7795458cef --- /dev/null +++ b/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n substring(path from '^f/([^/]+)/') AS \"folder!\",\n COUNT(*) AS \"script_count!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND path LIKE 'f/%'\n GROUP BY substring(path from '^f/([^/]+)/')\n ORDER BY substring(path from '^f/([^/]+)/')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "folder!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "script_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index ca95a3bca8..3efa843923 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -165,7 +165,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-ba7d01509662be936c000aab4ff60f57be839ae365e5cbbb4838a7b622d1771a.json b/backend/.sqlx/query-ba7d01509662be936c000aab4ff60f57be839ae365e5cbbb4838a7b622d1771a.json new file mode 100644 index 0000000000..b7752cc7cf --- /dev/null +++ b/backend/.sqlx/query-ba7d01509662be936c000aab4ff60f57be839ae365e5cbbb4838a7b622d1771a.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes)\n VALUES ('wm-fork-test-workspace', 'test-workspace', 'f/shared/other', 'script', 0, 1, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ba7d01509662be936c000aab4ff60f57be839ae365e5cbbb4838a7b622d1771a" +} diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index 045d470de5..f6ff25a4bf 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -79,7 +79,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index a8bb03e1b3..8dc66064dc 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -110,7 +110,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json new file mode 100644 index 0000000000..a051b068a5 --- /dev/null +++ b/backend/.sqlx/query-bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n VALUES ($1, $2, $3, $4, $5, 'script', $6) ON CONFLICT DO NOTHING\n RETURNING usage_access_type AS \"usage_access_type: AssetUsageAccessType\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage_access_type: AssetUsageAccessType", + "type_info": { + "Custom": { + "name": "asset_access_type", + "kind": { + "Enum": [ + "r", + "w", + "rw" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + }, + { + "Custom": { + "name": "asset_access_type", + "kind": { + "Enum": [ + "r", + "w", + "rw" + ] + } + } + }, + "Varchar", + "Jsonb" + ] + }, + "nullable": [ + true + ] + }, + "hash": "bfb97d2f48157a1575b7b6f3e64e0d075701d541a44dd7a0b586d1f337ced5e1" +} diff --git a/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json b/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json new file mode 100644 index 0000000000..db89d7bcfa --- /dev/null +++ b/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND runnable_path = $2 AND trigger_kind IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66" +} diff --git a/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json new file mode 100644 index 0000000000..7981dbc983 --- /dev/null +++ b/backend/.sqlx/query-c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3.json @@ -0,0 +1,111 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT asset_kind AS \"asset_kind: AssetKind\", asset_path, partition,\n status AS \"status: MaterializationStatus\", snapshot_id,\n row_count, job_id, materialized_at, error\n FROM materialized_partition\n WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3\n ORDER BY partition DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "partition", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "status: MaterializationStatus", + "type_info": { + "Custom": { + "name": "materialization_status", + "kind": { + "Enum": [ + "running", + "materialized", + "failed" + ] + } + } + } + }, + { + "ordinal": 4, + "name": "snapshot_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "row_count", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "job_id", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "materialized_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "error", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + true + ] + }, + "hash": "c28e066bf24263f9e3b48a2ecabdf037178ba246092dbc1d2b3816e7a9269dc3" +} diff --git a/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json b/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json new file mode 100644 index 0000000000..44e09f64d1 --- /dev/null +++ b/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET args = coalesce(args, '{}'::jsonb) || jsonb_build_object('partition', $1::text) WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828" +} diff --git a/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json b/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json new file mode 100644 index 0000000000..247d4de14a --- /dev/null +++ b/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by AS \"created_by!\"\n FROM v2_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae" +} diff --git a/backend/.sqlx/query-e0cc7528f34cca1a65bcff355805057b1c974a9a947bda133c155503dac1f545.json b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json similarity index 73% rename from backend/.sqlx/query-e0cc7528f34cca1a65bcff355805057b1c974a9a947bda133c155503dac1f545.json rename to backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json index c077375ef4..4629a9bd0a 100644 --- a/backend/.sqlx/query-e0cc7528f34cca1a65bcff355805057b1c974a9a947bda133c155503dac1f545.json +++ b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n VALUES ($1, $2, $3, $4, $5::text::json, now())\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = now()\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING created_at", + "query": "INSERT INTO draft (workspace_id, email, path, typ, value, created_at)\n VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now()))\n ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL\n DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at\n WHERE $7::bool = true\n OR $6::timestamptz IS NULL\n OR draft.created_at <= $6::timestamptz\n RETURNING created_at", "describe": { "columns": [ { @@ -49,12 +49,13 @@ }, "Text", "Timestamptz", - "Bool" + "Bool", + "Timestamptz" ] }, "nullable": [ false ] }, - "hash": "e0cc7528f34cca1a65bcff355805057b1c974a9a947bda133c155503dac1f545" + "hash": "c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7" } diff --git a/backend/.sqlx/query-ca1a39a56d36802418048ddad1301f16394c719ef55494b5f286bc402292c420.json b/backend/.sqlx/query-ca1a39a56d36802418048ddad1301f16394c719ef55494b5f286bc402292c420.json new file mode 100644 index 0000000000..5afd4065ae --- /dev/null +++ b/backend/.sqlx/query-ca1a39a56d36802418048ddad1301f16394c719ef55494b5f286bc402292c420.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork)\n VALUES ('test-workspace', 'wm-fork-test-workspace', 'f/shared/leaky', 'script', 1, 0, true, true, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "ca1a39a56d36802418048ddad1301f16394c719ef55494b5f286bc402292c420" +} diff --git a/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json b/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json new file mode 100644 index 0000000000..b5d905b9b6 --- /dev/null +++ b/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 5ed2e53367..d5365ffe94 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -110,7 +110,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index a27bea8b2a..33a5534b42 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -250,7 +250,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json b/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json new file mode 100644 index 0000000000..befd9903a9 --- /dev/null +++ b/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", trigger,\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc" +} diff --git a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json new file mode 100644 index 0000000000..2ba317edc6 --- /dev/null +++ b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b" +} diff --git a/backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json similarity index 64% rename from backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json rename to backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json index ef3968cdfa..037c796fc7 100644 --- a/backend/.sqlx/query-a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366.json +++ b/backend/.sqlx/query-e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING", + "query": "WITH ins AS (\n INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING\n RETURNING usage_kind, usage_access_type\n )\n INSERT INTO notify_event (channel, payload)\n SELECT 'notify_asset_producer_change', $1\n FROM ins WHERE usage_kind = 'script' AND usage_access_type IN ('w', 'rw')", "describe": { "columns": [], "parameters": { @@ -52,5 +52,5 @@ }, "nullable": [] }, - "hash": "a9e29764b5b9d94269e2b8aa755c71b61774c8ff8ae218d7a8d6ed0ac0169366" + "hash": "e041b20c1c4166b30ced8c6a0f50bcf0691ab2554ead6b489a8441a32fc0af82" } diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 1a4cc407e0..470c651020 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -190,7 +190,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json b/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json new file mode 100644 index 0000000000..d40882073d --- /dev/null +++ b/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path AS \"path!\" FROM script\n WHERE workspace_id = $1\n AND archived = false\n AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3" +} diff --git a/backend/.sqlx/query-eeaad5c2284c1856cfc64ae0bc0dfc79b283c06987a6de399b6c1aaca94a2b7a.json b/backend/.sqlx/query-eeaad5c2284c1856cfc64ae0bc0dfc79b283c06987a6de399b6c1aaca94a2b7a.json new file mode 100644 index 0000000000..169fe55e62 --- /dev/null +++ b/backend/.sqlx/query-eeaad5c2284c1856cfc64ae0bc0dfc79b283c06987a6de399b6c1aaca94a2b7a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO notify_event (channel, payload)\n VALUES ('notify_asset_producer_change', $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "eeaad5c2284c1856cfc64ae0bc0dfc79b283c06987a6de399b6c1aaca94a2b7a" +} diff --git a/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json b/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json new file mode 100644 index 0000000000..dd1db109da --- /dev/null +++ b/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json @@ -0,0 +1,51 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script_trigger\n (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all,\n debounce_s, retry_count, retry_delay_s)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "script_trigger_kind", + "kind": { + "Enum": [ + "asset", + "schedule", + "webhook", + "email", + "kafka", + "mqtt", + "nats", + "postgres", + "sqs", + "gcp" + ] + } + } + }, + "Text", + "Bool", + "Int4", + "Int2", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2" +} diff --git a/backend/.sqlx/query-fac563138c316998d4f523edd633db97dba77d649b637c2529e4f232dce16c88.json b/backend/.sqlx/query-fac563138c316998d4f523edd633db97dba77d649b637c2529e4f232dce16c88.json new file mode 100644 index 0000000000..9520c270dc --- /dev/null +++ b/backend/.sqlx/query-fac563138c316998d4f523edd633db97dba77d649b637c2529e4f232dce16c88.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id)\n WHERE j.parent_job = $1 AND q.running = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fac563138c316998d4f523edd633db97dba77d649b637c2529e4f232dce16c88" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index cdd9f6089e..63534975f5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -481,7 +481,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -493,7 +493,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -504,7 +504,7 @@ checksum = "0a184645bcc6f52d69d8e7639720699c6a99efb711f886e251ed1d16db8dd90e" dependencies = [ "quote", "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -645,7 +645,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -667,7 +667,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -678,7 +678,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1403,7 +1403,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1532,7 +1532,7 @@ dependencies = [ "regex", "rustc-hash 2.1.2", "shlex 1.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1552,7 +1552,7 @@ dependencies = [ "regex", "rustc-hash 2.1.2", "shlex 1.3.0", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1602,9 +1602,9 @@ dependencies = [ [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -1736,14 +1736,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "borsh" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a" +checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" dependencies = [ "borsh-derive", "bytes", @@ -1752,15 +1752,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59" +checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1770,7 +1770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1904,7 +1904,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1915,9 +1915,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] @@ -2042,7 +2042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2056,9 +2056,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -2205,7 +2205,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2469,9 +2469,9 @@ dependencies = [ [[package]] name = "cron" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "089df96cf6a25253b4b6b6744d86f91150a3d4df546f31a95def47976b8cba97" +checksum = "a5dcd6f69605c2956916ce24e8af637b754964c9a83f4662d3a2361654cdba09" dependencies = [ "chrono", "once_cell", @@ -2638,7 +2638,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2720,7 +2720,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2733,7 +2733,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2766,7 +2766,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2777,7 +2777,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3256,7 +3256,7 @@ checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3553,7 +3553,7 @@ checksum = "8380a4224d5d2c3f84da4d764c4326cac62e9a1e3d4960442d29136fc07be863" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3564,7 +3564,7 @@ checksum = "9b565e60a9685cdf312c888665b5f8647ac692a7da7e058a5e2268a466da8eaf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3733,7 +3733,7 @@ dependencies = [ "stringcase", "strum", "strum_macros", - "syn 2.0.117", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -3972,7 +3972,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3992,7 +3992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4021,7 +4021,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "unicode-xid", ] @@ -4034,7 +4034,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4160,7 +4160,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4183,7 +4183,7 @@ checksum = "0fbbb781877580993a8707ec48672673ec7b81eeba04cfd2310bd28c08e47c8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4309,7 +4309,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4366,7 +4366,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4386,7 +4386,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4406,7 +4406,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4679,7 +4679,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308530a56b099da144ebc5d8e179f343ad928fa2b3558d1eb3db9af18d6eff43" dependencies = [ "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4805,7 +4805,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5066,16 +5066,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", ] [[package]] @@ -5111,7 +5109,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5381,7 +5379,7 @@ dependencies = [ "indexmap 2.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5800,7 +5798,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -5993,12 +5991,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -6140,7 +6132,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6238,7 +6230,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6461,7 +6453,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6506,12 +6498,6 @@ dependencies = [ "spin 0.9.8", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -6969,7 +6955,7 @@ dependencies = [ "manyhow-macros", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7102,7 +7088,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7197,7 +7183,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7236,7 +7222,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "termcolor", "thiserror 2.0.18", ] @@ -7273,9 +7259,9 @@ dependencies = [ [[package]] name = "mysql_common" -version = "0.37.2" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2" +checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", "bitflags 2.13.0", @@ -7432,7 +7418,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7699,7 +7685,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7897,7 +7883,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8403,7 +8389,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8494,7 +8480,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8553,7 +8539,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8712,7 +8698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8764,7 +8750,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8776,7 +8762,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8872,7 +8858,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8928,9 +8914,9 @@ dependencies = [ [[package]] name = "pulp" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e205bb30d5b916c55e584c22201771bcf2bad9aabd5d4127f38387140c38632" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" dependencies = [ "bytemuck", "cfg-if", @@ -8945,9 +8931,9 @@ dependencies = [ [[package]] name = "pulp-wasm-simd-flag" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40e24eee682d89fb193496edf918a7f407d30175b2e785fe057e4392dfd182e0" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" [[package]] name = "pure-rust-locales" @@ -9112,7 +9098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -9309,7 +9295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9369,7 +9355,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9477,7 +9463,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -9689,7 +9675,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9728,7 +9714,7 @@ dependencies = [ "proc-macro2", "quote", "rquickjs-core", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9801,7 +9787,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.117", + "syn 2.0.118", "walkdir", ] @@ -10288,7 +10274,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10300,7 +10286,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10477,7 +10463,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10488,7 +10474,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10542,7 +10528,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10609,7 +10595,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10662,7 +10648,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10987,7 +10973,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11052,7 +11038,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11075,7 +11061,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.117", + "syn 2.0.118", "tokio", "url", ] @@ -11238,7 +11224,7 @@ checksum = "ae36a4951ca7bd1cfd991c241584a9824a70f6aff1e7d4f693fb3f2465e4030e" dependencies = [ "quote", "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11297,7 +11283,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11400,7 +11386,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11455,7 +11441,7 @@ checksum = "e276dc62c0a2625a560397827989c82a93fd545fcf6f7faec0935a82cc4ddbb8" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11556,7 +11542,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11661,7 +11647,7 @@ checksum = "c16ce73424a6316e95e09065ba6a207eba7765496fed113702278b7711d4b632" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11672,7 +11658,7 @@ checksum = "aae1efbaa74943dc5ad2a2fb16cbd78b77d7e4d63188f3c5b4df2b4dcd2faaae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11723,9 +11709,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -11749,7 +11735,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11778,7 +11764,7 @@ checksum = "181f22127402abcf8ee5c83ccd5b408933fec36a6095cf82cda545634692657e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12025,7 +12011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -12095,7 +12081,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12106,7 +12092,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12355,7 +12341,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12794,7 +12780,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13047,7 +13033,7 @@ checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13358,7 +13344,7 @@ version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -13460,16 +13446,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -13502,7 +13479,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -13537,7 +13514,7 @@ checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -13572,29 +13549,7 @@ checksum = "a369369e4360c2884c3168d22bded735c43cccae97bbc147586d4b480edd138d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", -] - -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", + "syn 2.0.118", ] [[package]] @@ -13620,18 +13575,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.13.0", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver 1.0.28", -] - [[package]] name = "wasmtimer" version = "0.4.3" @@ -13680,9 +13623,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -13693,14 +13636,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -13792,7 +13735,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-nats", @@ -13874,7 +13817,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.727.0" +version = "1.734.0" dependencies = [ "async-stream", "async-trait", @@ -13907,7 +13850,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13920,7 +13863,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "argon2", @@ -14058,7 +14001,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14081,7 +14024,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14094,7 +14037,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14120,7 +14063,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.727.0" +version = "1.734.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14130,7 +14073,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14147,7 +14090,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14169,7 +14112,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14192,7 +14135,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14151,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14229,7 +14172,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14250,7 +14193,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14264,7 +14207,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-nats", @@ -14299,7 +14242,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14324,7 +14267,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14342,7 +14285,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14364,7 +14307,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14384,7 +14327,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14408,14 +14351,20 @@ dependencies = [ "windmill-dep-map", "windmill-git-sync", "windmill-object-store", + "windmill-parser", "windmill-parser-py", + "windmill-parser-py-asset", + "windmill-parser-sql", + "windmill-parser-sql-asset", "windmill-parser-ts", + "windmill-parser-ts-asset", + "windmill-parser-yaml", "windmill-queue", ] [[package]] name = "windmill-api-settings" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14443,7 +14392,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.727.0" +version = "1.734.0" dependencies = [ "lazy_static", "serde", @@ -14455,7 +14404,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.727.0" +version = "1.734.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14480,7 +14429,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14494,7 +14443,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.727.0" +version = "1.734.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14527,7 +14476,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.727.0" +version = "1.734.0" dependencies = [ "chrono", "lazy_static", @@ -14541,7 +14490,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14560,7 +14509,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.727.0" +version = "1.734.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14662,7 +14611,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.727.0" +version = "1.734.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14681,7 +14630,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.727.0" +version = "1.734.0" dependencies = [ "regex", "serde", @@ -14696,7 +14645,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14720,7 +14669,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "futures", @@ -14737,7 +14686,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.727.0" +version = "1.734.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14748,12 +14697,12 @@ dependencies = [ "serde", "serde_derive", "serde_yml", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] name = "windmill-mcp" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -14774,7 +14723,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -14805,7 +14754,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "arc-swap", @@ -14830,7 +14779,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-stream", @@ -14864,7 +14813,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "futures", @@ -14882,7 +14831,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.727.0" +version = "1.734.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14891,7 +14840,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -14903,7 +14852,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -14915,7 +14864,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "gosyn", @@ -14927,7 +14876,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -14939,7 +14888,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -14951,7 +14900,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "nu-parser", @@ -14962,7 +14911,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14973,7 +14922,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14985,7 +14934,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14996,7 +14945,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-recursion", @@ -15018,7 +14967,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -15030,7 +14979,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -15044,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15054,14 +15003,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.117", + "syn 2.0.118", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -15074,7 +15023,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde", @@ -15086,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -15104,7 +15053,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15120,7 +15069,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15136,7 +15085,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde", @@ -15147,7 +15096,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-recursion", @@ -15185,7 +15134,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "const_format", @@ -15224,7 +15173,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.727.0" +version = "1.734.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15235,7 +15184,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-recursion", @@ -15267,7 +15216,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15291,7 +15240,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15324,7 +15273,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15357,7 +15306,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15377,7 +15326,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15411,7 +15360,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15396,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15470,7 +15419,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15494,7 +15443,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-nats", @@ -15518,7 +15467,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15553,7 +15502,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15581,7 +15530,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-trait", @@ -15606,7 +15555,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15625,7 +15574,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-once-cell", @@ -15735,7 +15684,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.727.0" +version = "1.734.0" dependencies = [ "bytes", "futures", @@ -15862,7 +15811,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15873,7 +15822,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15884,7 +15833,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15895,7 +15844,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15906,7 +15855,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15917,7 +15866,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -16348,100 +16297,12 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.117", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.117", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.13.0", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver 1.0.28", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" @@ -16543,7 +16404,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -16564,7 +16425,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -16584,7 +16445,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure", ] @@ -16624,7 +16485,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index de302ce748..a4dd1ce96f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.727.0" +version = "1.734.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.727.0" +version = "1.734.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index 8cc71ec71c..dc98a5a625 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -97,7 +97,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence | |---|---|---|---|---|---|---|---|---|---| | T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b | -| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | +| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | | T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 | | T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 | | T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c625d6bc09..b3422ed267 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0efd966206cf914caae0a9be9992bfa55cdb7a25 +1454aaa9e60e17cfb3c22002594900dbaca38455 diff --git a/backend/migrations/20260423050000_script_trigger.down.sql b/backend/migrations/20260423050000_script_trigger.down.sql new file mode 100644 index 0000000000..5171e8f2d0 --- /dev/null +++ b/backend/migrations/20260423050000_script_trigger.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_script_pipeline_path; +DROP TABLE IF EXISTS script_trigger; +DROP TYPE IF EXISTS SCRIPT_TRIGGER_KIND; diff --git a/backend/migrations/20260423050000_script_trigger.up.sql b/backend/migrations/20260423050000_script_trigger.up.sql new file mode 100644 index 0000000000..cb105ba3f3 --- /dev/null +++ b/backend/migrations/20260423050000_script_trigger.up.sql @@ -0,0 +1,58 @@ +-- Execution DAG edges declared via `// on ` +-- annotations. +-- For `trigger_kind='asset'`: trigger_ref is `://` (kind from +-- parse_asset_syntax, so downstream lookups match the `asset` table). +-- The other kinds mirror the keywords the annotation parser recognises in +-- `// on ` lines (every non-integration trigger kind; their +-- trigger_ref is the trigger row path, or empty for marker-only forms). +-- +-- Per-edge columns that are in fact script-level properties (every row for +-- a given runnable carries the same value, set once at deploy) but live on +-- the edge so the dispatcher reads everything from a single query: +-- join_all `// trigger all` AND-join barrier (else OR, the default). +-- debounce_s `// on debounce=` (else script-level `// debounce`); only +-- asset-cascade edges carry one. NULL = no debounce. +-- retry_* `// retry []` cascade retry. NULL = none. +-- +-- The idempotency guards (IF NOT EXISTS / duplicate_object) are load-bearing: +-- this migration squashes several pre-release ones, so databases migrated +-- from the unsquashed history already contain the final objects and +-- re-applying must be a no-op. +DO $$ BEGIN + CREATE TYPE SCRIPT_TRIGGER_KIND AS ENUM ( + 'asset', 'schedule', 'webhook', 'email', 'kafka', 'mqtt', 'nats', + 'postgres', 'sqs', 'gcp'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS script_trigger ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + runnable_kind ASSET_USAGE_KIND NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + trigger_kind SCRIPT_TRIGGER_KIND NOT NULL, + trigger_ref TEXT NOT NULL, + join_all BOOLEAN NOT NULL DEFAULT FALSE, + debounce_s INTEGER, + retry_count SMALLINT, + retry_delay_s INTEGER +); + +-- Per-runnable lookup (wipe-on-deploy, list-triggers-for-script). +CREATE INDEX IF NOT EXISTS idx_script_trigger_runnable + ON script_trigger (workspace_id, runnable_kind, runnable_path); + +-- Reverse lookup: "which scripts are triggered by asset X?" (the asset → script +-- edges in the graph). trigger_ref is unbounded text so can't share the +-- asset_kind btree, but this covers the common prefix-scan use case. +CREATE INDEX IF NOT EXISTS idx_script_trigger_ref + ON script_trigger (workspace_id, trigger_kind, trigger_ref); + +-- Fast lookups for: +-- 1. "does folder F have a pipeline?" (exists check on prefix) +-- 2. "list all folders with a pipeline" (distinct folder from path) +-- The partial predicate keeps the index tiny on workspaces with few +-- pipeline scripts, and text_pattern_ops lets 'f/foo/%' LIKE scans use it. +CREATE INDEX IF NOT EXISTS idx_script_pipeline_path + ON script (workspace_id, path text_pattern_ops) + WHERE auto_kind = 'pipeline' AND archived = false AND deleted = false; diff --git a/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql b/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql new file mode 100644 index 0000000000..76e5b7050c --- /dev/null +++ b/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql @@ -0,0 +1,3 @@ +-- Postgres has no ALTER TYPE ... DROP VALUE for enums. The 'asset' value +-- stays even on rollback; this is consistent with how other job_trigger_kind +-- values were added (see 20250323162033_add-missing-trigger-kind-...). diff --git a/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql b/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql new file mode 100644 index 0000000000..30cdb58ec3 --- /dev/null +++ b/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql @@ -0,0 +1,5 @@ +-- Add 'asset' as a job_trigger_kind so jobs that get dispatched as a +-- consequence of an upstream pipeline script writing an asset can be +-- attributed via v2_job.trigger_kind = 'asset'. The producer's runnable +-- path goes into v2_job.trigger. +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'asset'; diff --git a/backend/migrations/20260516194247_join_pending_inputs.down.sql b/backend/migrations/20260516194247_join_pending_inputs.down.sql new file mode 100644 index 0000000000..6e67e88e1b --- /dev/null +++ b/backend/migrations/20260516194247_join_pending_inputs.down.sql @@ -0,0 +1 @@ +DROP TABLE join_pending_inputs; diff --git a/backend/migrations/20260516194247_join_pending_inputs.up.sql b/backend/migrations/20260516194247_join_pending_inputs.up.sql new file mode 100644 index 0000000000..cbda6dd16e --- /dev/null +++ b/backend/migrations/20260516194247_join_pending_inputs.up.sql @@ -0,0 +1,19 @@ +-- AND-join barrier slot state. For a `// trigger all` subscriber, each +-- partition-bearing input arrival (an `// on` asset whose declared path +-- contains the `{partition}` token) is recorded against the +-- (subscriber, partition) slot. The subscriber is dispatched once, for a +-- given partition, only when every partition-bearing input it declares has +-- arrived for that partition — skew-immune (unlike a debounce). The slot +-- is cleared on fire so later writes re-accumulate and can re-materialize. +-- +-- trigger_ref stores the literal `{partition}`-token form (lineage is +-- partition-agnostic; the concrete value is the `partition` column), +-- matching how script_trigger / asset rows store it. +CREATE TABLE join_pending_inputs ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + subscriber_path VARCHAR(255) NOT NULL, + partition TEXT NOT NULL, + trigger_ref TEXT NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, subscriber_path, partition, trigger_ref) +); diff --git a/backend/migrations/20260523055641_dispatch_event.down.sql b/backend/migrations/20260523055641_dispatch_event.down.sql new file mode 100644 index 0000000000..c6fdefc07c --- /dev/null +++ b/backend/migrations/20260523055641_dispatch_event.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dispatch_event; +DROP TYPE IF EXISTS DISPATCH_OUTCOME; diff --git a/backend/migrations/20260523055641_dispatch_event.up.sql b/backend/migrations/20260523055641_dispatch_event.up.sql new file mode 100644 index 0000000000..bf68281fbd --- /dev/null +++ b/backend/migrations/20260523055641_dispatch_event.up.sql @@ -0,0 +1,63 @@ +-- Per-decision log of what the asset-trigger dispatcher did after each +-- producer job completed. One row per (producer, subscriber, asset write) +-- decision: dispatched, debounced, join_pending (partial AND-join), or +-- skipped (with reason). Surfaced on the producer's job detail page so +-- the cascade is no longer invisible when the producer "succeeds" but +-- no child appears. +-- +-- Retention: the FK to v2_job(id) ON DELETE CASCADE means the existing +-- retention sweep (monitor.rs delete_expired_jobs_batch -> DELETE FROM +-- v2_job WHERE id = ANY(...)) reaps these rows along with their producer. +-- No separate cleanup path needed. +-- +-- Idempotency guards (duplicate_object / IF NOT EXISTS) are load-bearing: +-- re-applying this migration after a squash must be a no-op. +DO $$ BEGIN + CREATE TYPE DISPATCH_OUTCOME AS ENUM ( + 'dispatched', + 'join_pending', + 'skipped' + ); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS dispatch_event ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + producer_job_id UUID NOT NULL REFERENCES v2_job(id) ON DELETE CASCADE, + subscriber_path VARCHAR(255) NOT NULL, + asset_kind ASSET_KIND NOT NULL, + asset_path TEXT NOT NULL, + outcome DISPATCH_OUTCOME NOT NULL, + -- Set for 'dispatched'. Intentionally not FK'd: the subscriber job may + -- be retention-reaped independently, and we still want the row to + -- record "we dispatched " historically (UI renders a dead link). + child_job_id UUID, + partition TEXT, + -- AND-join progress at decision time. NULL for non-join subscribers. + received_inputs INTEGER, + required_inputs INTEGER, + -- Effective debounce window applied to this dispatch (NULL = none). + debounce_s INTEGER, + -- Free-text discriminator for 'skipped' outcomes (self_loop, + -- case3_non_partition_bearing, case3_missing_partition, cycle_detected, ...). + reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Primary access pattern: list events for one producer (the job detail +-- panel). Ordered scans by id give chronological order for free. +CREATE INDEX IF NOT EXISTS idx_dispatch_event_producer + ON dispatch_event (producer_job_id, id); + +-- Backs the asset-graph edge listing (jobs.rs list_asset_dispatch_edges): +-- WHERE workspace_id = $1 AND subscriber_path LIKE 'prefix%' +-- AND created_at >= $3 +-- ORDER BY created_at DESC, id DESC +-- The (producer_job_id, id) index above doesn't help this access path, so +-- without this one a high-volume dispatch_event seq-scans + sorts. +-- text_pattern_ops makes the anchored LIKE prefix (built as `path_start || '%'`) +-- index-usable regardless of the column collation; created_at DESC matches +-- the ORDER BY so Postgres can satisfy ordering from the index. +CREATE INDEX IF NOT EXISTS idx_dispatch_event_subscriber + ON dispatch_event (workspace_id, subscriber_path text_pattern_ops, created_at DESC); diff --git a/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql new file mode 100644 index 0000000000..fcb5ef5bbf --- /dev/null +++ b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql @@ -0,0 +1,2 @@ +-- Postgres cannot drop a single enum value; leaving 'data_pipeline' in +-- DRAFT_KIND is harmless on rollback. diff --git a/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql new file mode 100644 index 0000000000..38ae68ddcb --- /dev/null +++ b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql @@ -0,0 +1,6 @@ +-- A `data_pipeline` draft bundles every unsaved pipeline script of a folder +-- into a single row keyed at the folder path (typ has no deployed backing +-- table — see UserDraftItemKind::deployed_table). Lets the asset-graph view +-- store its in-flight drafts in the per-user DB draft sync instead of +-- browser-local storage. +ALTER TYPE DRAFT_KIND ADD VALUE IF NOT EXISTS 'data_pipeline'; diff --git a/backend/migrations/20260616120048_backfill_legacy_draft_emails.down.sql b/backend/migrations/20260616120048_backfill_legacy_draft_emails.down.sql new file mode 100644 index 0000000000..a0ef6d0cfe --- /dev/null +++ b/backend/migrations/20260616120048_backfill_legacy_draft_emails.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data backfill: once an email is attached, the row is +-- indistinguishable from a draft that was always per-user owned, so the +-- original NULL state cannot be reconstructed. No-op on revert. diff --git a/backend/migrations/20260616120048_backfill_legacy_draft_emails.up.sql b/backend/migrations/20260616120048_backfill_legacy_draft_emails.up.sql new file mode 100644 index 0000000000..cdf4b19c79 --- /dev/null +++ b/backend/migrations/20260616120048_backfill_legacy_draft_emails.up.sql @@ -0,0 +1,28 @@ +-- Backfill the owner `email` on legacy drafts (rows persisted before per-user +-- sync, hence `email IS NULL`). A user-owned draft path is `u//...`, +-- so resolve `` against `usr` for the same workspace and adopt that +-- user's email. +-- +-- Guards: +-- - the resolved email must exist in `password` (the `draft_password_fkey` +-- target), or the UPDATE would violate the FK; +-- - skip rows that would collide with an existing per-user draft at the same +-- (workspace_id, path, typ, email) under the `draft_pkey_with_user` partial +-- unique index — the per-user row is authoritative, so the legacy row is +-- left untouched. +UPDATE draft d +SET email = u.email +FROM usr u +WHERE d.email IS NULL + AND split_part(d.path, '/', 1) = 'u' + AND split_part(d.path, '/', 2) <> '' + AND u.workspace_id = d.workspace_id + AND u.username = split_part(d.path, '/', 2) + AND EXISTS (SELECT 1 FROM password p WHERE p.email = u.email) + AND NOT EXISTS ( + SELECT 1 FROM draft d2 + WHERE d2.workspace_id = d.workspace_id + AND d2.path = d.path + AND d2.typ = d.typ + AND d2.email = u.email + ); diff --git a/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.down.sql b/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.down.sql new file mode 100644 index 0000000000..6e1600b72a --- /dev/null +++ b/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data cleanup: deleted orphaned rows and reset cached verdicts +-- cannot be reconstructed. No-op on rollback. +SELECT 1; diff --git a/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.up.sql b/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.up.sql new file mode 100644 index 0000000000..2d4f82ab83 --- /dev/null +++ b/backend/migrations/20260616210529_cleanup_orphaned_workspace_diff.up.sql @@ -0,0 +1,31 @@ +-- workspace_diff / skip_workspace_diff_tally are keyed by workspace id with no FK +-- cascade, so until the matching delete_workspace cleanup landed, deleting a fork +-- left its cached diff rows behind. Workspace ids are reused (recreating a fork +-- under the same name), so those orphaned rows leaked onto the new fork and +-- produced a spurious "changes not visible" warning that hid the deploy button. + +-- Drop rows referencing workspaces that no longer exist (leftovers from past deletes). +DELETE FROM workspace_diff +WHERE source_workspace_id NOT IN (SELECT id FROM workspace) + OR fork_workspace_id NOT IN (SELECT id FROM workspace); + +DELETE FROM skip_workspace_diff_tally +WHERE workspace_id NOT IN (SELECT id FROM workspace); + +-- Reused-id victims keep a skip-tally row pointing at a LIVE workspace, so the +-- orphan cleanup above can't reach them. compare_workspaces consults +-- skip_workspace_diff_tally first and short-circuits to an empty comparison, so a +-- stale skip row would also defeat the has_changes reset below. A genuinely +-- skipped workspace is excluded from tallying and therefore never has +-- workspace_diff rows — so a skip row that coexists with workspace_diff rows for +-- that id is provably leaked from a previous occupant of a reused id. Drop those. +DELETE FROM skip_workspace_diff_tally s +WHERE EXISTS ( + SELECT 1 FROM workspace_diff d WHERE d.fork_workspace_id = s.workspace_id +); + +-- Remaining reused-id victims keep workspace_diff rows pointing at live +-- workspaces, so they can't be told apart from valid cache. Reset the cached +-- verdict to force a recompute on the next compare; compare_workspaces +-- re-evaluates NULL rows and corrects or deletes them. +UPDATE workspace_diff SET has_changes = NULL; diff --git a/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.down.sql b/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.down.sql new file mode 100644 index 0000000000..7d29137db3 --- /dev/null +++ b/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.down.sql @@ -0,0 +1,4 @@ +REVOKE ALL ON notify_event FROM windmill_user; +REVOKE ALL ON notify_event FROM windmill_admin; +REVOKE ALL ON SEQUENCE notify_event_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE notify_event_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.up.sql b/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.up.sql new file mode 100644 index 0000000000..0d0ac84824 --- /dev/null +++ b/backend/migrations/20260619091631_grant_notify_event_to_windmill_roles.up.sql @@ -0,0 +1,14 @@ +-- The notify_event table (migration 20260203172950_polling_based_events) was +-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user +-- and windmill_admin. Those default privileges only apply to objects created by +-- the role that set them (migration 20250205131523), so deployments whose +-- migration runner is a different role leave notify_event ungranted. Trigger +-- inserts were worked around with SECURITY DEFINER, but direct application +-- inserts (clear_static_asset_usage in assets.rs, restart_worker_group in +-- settings) run as the invoking role and fail with "permission denied for table +-- notify_event". Grant explicitly to guarantee access regardless of who ran the +-- migrations. +GRANT ALL ON notify_event TO windmill_user; +GRANT ALL ON notify_event TO windmill_admin; +GRANT ALL ON SEQUENCE notify_event_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE notify_event_id_seq TO windmill_admin; diff --git a/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.down.sql b/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.down.sql new file mode 100644 index 0000000000..c84fe703bb --- /dev/null +++ b/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.down.sql @@ -0,0 +1,4 @@ +REVOKE ALL ON script_trigger FROM windmill_user; +REVOKE ALL ON script_trigger FROM windmill_admin; +REVOKE ALL ON SEQUENCE script_trigger_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE script_trigger_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.up.sql b/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.up.sql new file mode 100644 index 0000000000..1031f22a6a --- /dev/null +++ b/backend/migrations/20260619112847_grant_script_trigger_to_windmill_roles.up.sql @@ -0,0 +1,14 @@ +-- The script_trigger table (migration 20260423050000_script_trigger) was +-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user +-- and windmill_admin. Those default privileges only apply to objects created by +-- the role that set them (migration 20250205131523), so deployments whose +-- migration runner is a different role leave script_trigger ungranted. Direct +-- application writes run as the invoking role (clear_script_triggers and +-- insert_script_trigger in windmill-common/src/assets.rs, every script save) +-- and fail with "permission denied for table script_trigger". Grant explicitly +-- to guarantee access regardless of who ran the migrations (same fix as +-- notify_event in 20260619091631). +GRANT ALL ON script_trigger TO windmill_user; +GRANT ALL ON script_trigger TO windmill_admin; +GRANT ALL ON SEQUENCE script_trigger_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE script_trigger_id_seq TO windmill_admin; diff --git a/backend/migrations/20260619113554_scrub_draft_value_nul.down.sql b/backend/migrations/20260619113554_scrub_draft_value_nul.down.sql new file mode 100644 index 0000000000..098a525396 --- /dev/null +++ b/backend/migrations/20260619113554_scrub_draft_value_nul.down.sql @@ -0,0 +1,2 @@ +-- Irreversible: a stripped NUL cannot be restored (and was never meaningful). +SELECT 1; diff --git a/backend/migrations/20260619113554_scrub_draft_value_nul.up.sql b/backend/migrations/20260619113554_scrub_draft_value_nul.up.sql new file mode 100644 index 0000000000..1ae927bae6 --- /dev/null +++ b/backend/migrations/20260619113554_scrub_draft_value_nul.up.sql @@ -0,0 +1,30 @@ +-- One-time cleanup of drafts whose `json` value carries a real U+0000 (NUL) +-- escape — storable only because `draft.value` is `json`, not `jsonb`. Such a +-- value makes any `->>`/`to_jsonb` extraction raise `22P05`, which 500'd +-- GET /drafts/list. New writes are sanitized in the application layer +-- (update_draft → strip_json_nul); this fixes rows written before that landed. +-- +-- Only genuinely-poisoned rows are touched: a real NUL makes `value::jsonb` +-- raise, which distinguishes it from a legitimately escaped backslash sequence +-- (which `jsonb` accepts). The text replace handles the real-world shape — a NUL +-- inside a text field. A contrived value where stripping the escape leaves +-- invalid JSON is left as-is (and can no longer be created). +DO $$ +DECLARE r RECORD; +BEGIN + FOR r IN + SELECT id, value FROM draft WHERE position(E'\\u0000' in value::text) > 0 + LOOP + BEGIN + PERFORM r.value::jsonb; -- not poisoned (legit escaped backslash): skip + EXCEPTION WHEN others THEN + BEGIN + UPDATE draft + SET value = replace(r.value::text, E'\\u0000', '')::json + WHERE id = r.id; + EXCEPTION WHEN others THEN + NULL; -- pathological shape; cannot strip in SQL, no longer creatable + END; + END; + END LOOP; +END $$; diff --git a/backend/migrations/20260619170118_add_materialized_partition.down.sql b/backend/migrations/20260619170118_add_materialized_partition.down.sql new file mode 100644 index 0000000000..4932338956 --- /dev/null +++ b/backend/migrations/20260619170118_add_materialized_partition.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_materialized_partition_asset_status; +DROP TABLE IF EXISTS materialized_partition; +DROP TYPE IF EXISTS MATERIALIZATION_STATUS; diff --git a/backend/migrations/20260619170118_add_materialized_partition.up.sql b/backend/migrations/20260619170118_add_materialized_partition.up.sql new file mode 100644 index 0000000000..da5f806e4f --- /dev/null +++ b/backend/migrations/20260619170118_add_materialized_partition.up.sql @@ -0,0 +1,29 @@ +-- Per-partition materialization state for managed `// materialize` assets. +-- One row per (asset, partition): the latest materialization of that slice. +-- Drives: the partition-status grid (CE observability), run-stale/gap +-- detection, and the EE backfill worklist (missing/failed partitions). The +-- `partition` column uses '' as the sentinel for an unpartitioned (whole-table) +-- materialization, since partition is part of the primary key and cannot be +-- NULL. +CREATE TYPE MATERIALIZATION_STATUS AS ENUM ('running', 'materialized', 'failed'); + +CREATE TABLE IF NOT EXISTS materialized_partition ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + asset_kind ASSET_KIND NOT NULL, + asset_path VARCHAR(255) NOT NULL, + partition TEXT NOT NULL DEFAULT '', + status MATERIALIZATION_STATUS NOT NULL, + -- DuckLake snapshot id produced by the write; NULL while running / on + -- failure. The pin that makes downstream reads reproducible. + snapshot_id BIGINT, + row_count BIGINT, + job_id UUID, + materialized_at TIMESTAMPTZ NOT NULL DEFAULT now(), + error TEXT, + PRIMARY KEY (workspace_id, asset_kind, asset_path, partition) +); + +-- Backfill enumeration / grid "show only gaps": filter an asset's partitions +-- by status without scanning the whole table. +CREATE INDEX IF NOT EXISTS idx_materialized_partition_asset_status + ON materialized_partition (workspace_id, asset_kind, asset_path, status); diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 87cddda16b..9e74822d98 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -12,6 +12,7 @@ "bitbucket": { "auth_url": "https://bitbucket.org/site/oauth2/authorize", "token_url": "https://bitbucket.org/site/oauth2/access_token", + "grant_types": ["authorization_code", "client_credentials"], "scopes": ["repository"] }, "slack": { @@ -103,6 +104,7 @@ "linkedin": { "auth_url": "https://www.linkedin.com/oauth/v2/authorization", "token_url": "https://www.linkedin.com/oauth/v2/accessToken", + "grant_types": ["authorization_code", "client_credentials"], "scopes": ["w_member_social", "r_liteprofile", "r_emailaddress"], "req_body_auth": true }, @@ -114,14 +116,46 @@ "visma": { "auth_url": "https://connect.visma.com/connect/authorize", "token_url": "https://connect.visma.com/connect/token", + "grant_types": ["authorization_code", "client_credentials"], "scopes": [ "offline_access", "vismanet_erp_interactive_api:create", "vismanet_erp_interactive_api:delete", "vismanet_erp_interactive_api:read", "vismanet_erp_interactive_api:update" + ], + "cc_scopes": [ + "vismanet_erp_service_api:create", + "vismanet_erp_service_api:delete", + "vismanet_erp_service_api:read", + "vismanet_erp_service_api:update" ] }, + "coupa": { + "grant_types": ["client_credentials"], + "cc_scopes": [ + "core.supplier.read", + "core.supplier.write", + "core.purchase_order.read", + "core.purchase_order.write", + "core.requisition.read", + "core.requisition.write", + "core.invoice.read", + "core.invoice.write", + "core.contract.read", + "core.expense.read" + ], + "connect_config_template": { + "display_name": "Coupa", + "label": "Coupa instance", + "placeholder": "your-instance", + "token_url": "https://{instance}.coupahost.com/oauth2/token", + "strip_suffix": ".coupahost.com", + "resource_mapping": { + "instance_url": "https://{instance}.coupahost.com" + } + } + }, "sage_intacct": { "auth_url": "https://api.intacct.com/ia/api/v1/oauth2/authorize", "token_url": "https://api.intacct.com/ia/api/v1/oauth2/token", @@ -130,6 +164,7 @@ "spotify": { "auth_url": "https://accounts.spotify.com/authorize", "token_url": "https://accounts.spotify.com/api/token", + "grant_types": ["authorization_code", "client_credentials"], "scopes": [ "user-read-playback-state", "user-modify-playback-state", @@ -149,12 +184,16 @@ "xero": { "auth_url": "https://login.xero.com/identity/connect/authorize", "token_url": "https://identity.xero.com/connect/token", - "scopes": ["offline_access", "accounting.transactions"] + "grant_types": ["authorization_code", "client_credentials"], + "scopes": ["offline_access", "accounting.transactions"], + "cc_scopes": ["accounting.transactions"] }, "zoho": { "auth_url": "https://accounts.zoho.com/oauth/v2/auth", "token_url": "https://accounts.zoho.com/oauth/v2/token", + "grant_types": ["authorization_code", "client_credentials"], "scopes": ["ZohoAssist.sessionapi.ALL"], + "cc_scopes": ["ZohoAssist.sessionapi.ALL"], "extra_params": { "access_type": "offline" } @@ -197,6 +236,8 @@ } }, "servicenow": { + "grant_types": ["authorization_code", "client_credentials"], + "req_body_auth": true, "connect_config_template": { "display_name": "ServiceNow", "label": "ServiceNow Instance", @@ -211,6 +252,23 @@ } } }, + "netsuite": { + "connect_config_template": { + "display_name": "NetSuite", + "label": "NetSuite Account ID", + "placeholder": " as in your NetSuite domain, e.g. 1234567 or 1234567-sb1", + "help_url": "https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/chapter_157769826287.html", + "auth_url": "https://{instance}.app.netsuite.com/app/login/oauth2/authorize.nl", + "token_url": "https://{instance}.suitetalk.api.netsuite.com/services/rest/auth/oauth2/v1/token", + "req_body_auth": false, + "strip_suffix": ".app.netsuite.com", + "scopes": ["rest_webservices"], + "extra_params_key": "account_id", + "resource_mapping": { + "account_id": "{instance}" + } + } + }, "outreach": { "auth_url": "https://api.outreach.io/oauth/authorize", "token_url": "https://api.outreach.io/oauth/token", diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 10b93a998b..e8a0a1cc40 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -2,8 +2,8 @@ use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; use rustpython_parser::{ast::Suite, Parse}; use std::collections::HashMap; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -28,7 +28,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(assets_finder.assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(assets_finder.assets), + Vec::new(), + pipeline, + )) } type VarAssetName = String; diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index 002ed78fc0..812bfa785d 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -9,8 +9,8 @@ use sqlparser::{ parser::Parser, }; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -33,7 +33,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(collector.assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(collector.assets), + Vec::new(), + pipeline, + )) } /// Visitor that collects S3 asset literals from SQL statements @@ -260,6 +265,21 @@ impl AssetCollector { } } + // Collect the table-level reads (and column assets) of a query's top + // SELECT. Table-level reads are only gathered here and in the statement + // arms — the generic table-factor visitor picks up read-functions and + // string literals, not plain `FROM ` references. Called for both + // standalone SELECTs and the `AS SELECT` of CTAS / CREATE VIEW. + fn handle_query_reads(&mut self, query: &sqlparser::ast::Query) { + self.cte_name_stack.push(collect_cte_names(query)); + if let Some(select) = query.body.as_select() { + for t in &select.from { + self.handle_table_with_joins(t, Some(R)); + } + self.extract_column_assets(&select.projection, &select.from); + } + } + fn handle_table_with_joins( &mut self, table_with_joins: &sqlparser::ast::TableWithJoins, @@ -485,15 +505,7 @@ impl Visitor for AssetCollector { ) -> std::ops::ControlFlow { match statement { sqlparser::ast::Statement::Query(q) => { - self.cte_name_stack.push(collect_cte_names(q)); - if let Some(select) = q.body.as_select() { - // First, handle table references (adds table-level assets) - for t in &select.from { - self.handle_table_with_joins(t, Some(R)); - } - // Then, extract column-level assets - self.extract_column_assets(&select.projection, &select.from); - } + self.handle_query_reads(q); } sqlparser::ast::Statement::Insert(insert) => { @@ -647,10 +659,37 @@ impl Visitor for AssetCollector { sqlparser::ast::Statement::CreateTable(create_table) => { self.track_table_definition(&create_table.name); + // `CREATE TABLE x AS SELECT … FROM y` reads y. The AS-query + // isn't a `Statement::Query`, so its FROM tables are only + // caught here. + if let Some(query) = &create_table.query { + self.handle_query_reads(query); + } } - sqlparser::ast::Statement::CreateView { name, .. } => { + sqlparser::ast::Statement::CreateView { name, query, .. } => { self.track_table_definition(name); + self.handle_query_reads(query); + } + + // DROP TABLE/VIEW is a write to the dropped object — the + // canonical idempotent-refresh pattern (`DROP TABLE IF EXISTS x; + // CREATE TABLE x AS …`) must resolve to a table-level write. + // Without this arm, a tagged-template snippet containing only the + // DROP yields no table-level asset and the TS/Python SDK parsers + // fall back to a db-level `datatable://` reference, putting a + // stray database node on the pipeline canvas. + sqlparser::ast::Statement::Drop { object_type, names, .. } => { + if matches!( + object_type, + sqlparser::ast::ObjectType::Table + | sqlparser::ast::ObjectType::View + | sqlparser::ast::ObjectType::MaterializedView + ) { + for name in names { + self.track_table_definition(name); + } + } } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { @@ -702,7 +741,16 @@ impl Visitor for AssetCollector { &mut self, statement: &sqlparser::ast::Statement, ) -> std::ops::ControlFlow { - if matches!(statement, sqlparser::ast::Statement::Query(_)) { + // Balance the push done by handle_query_reads (called from the Query, + // CreateView, and CTAS arms). + let pushed = match statement { + sqlparser::ast::Statement::Query(_) | sqlparser::ast::Statement::CreateView { .. } => { + true + } + sqlparser::ast::Statement::CreateTable(ct) => ct.query.is_some(), + _ => false, + }; + if pushed { self.cte_name_stack.pop(); } std::ops::ControlFlow::Continue(()) @@ -926,6 +974,61 @@ mod tests { ); } + #[test] + fn test_sql_asset_parser_drop_table_is_write() { + // A lone DROP (e.g. one SDK tagged-template snippet of an + // idempotent-refresh script) must resolve to a table-level write, + // not fall through to nothing. + let input = r#" + ATTACH 'datatable://main' AS dt; USE dt; + DROP TABLE IF EXISTS orders_raw; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/orders_raw".to_string(), + access_type: Some(W), + columns: None + },]) + ); + } + + #[test] + fn test_sql_asset_parser_drop_then_create_qualified() { + // Canonical refresh pattern over an attached catalog: DROP + CREATE + // AS on the output, SELECT on the input. + let input = r#" + ATTACH 'datatable://main' AS pg; + DROP TABLE IF EXISTS pg.daily_revenue; + CREATE TABLE pg.daily_revenue AS SELECT * FROM pg.orders_clean; + "#; + let s = parse_assets(input).map(|mut s| { + s.assets.sort_by(|a, b| a.path.cmp(&b.path)); + s.assets + }); + // The DROP+CTAS combo yields a clean write of the output plus the + // read of the CTAS source (`SELECT * FROM pg.orders_clean`). + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/daily_revenue".to_string(), + access_type: Some(W), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/orders_clean".to_string(), + access_type: Some(R), + columns: None + }, + ]) + ); + } + // Make sure a_function is not detected as main/a_function #[test] fn test_sql_asset_parser_function_table() { @@ -1796,3 +1899,45 @@ mod tests { assert_eq!(columns.get("age"), Some(&R)); } } + +#[cfg(test)] +mod ctas_read_tests { + use super::*; + + #[test] + fn test_ctas_collects_upstream_read() { + let input = r#" + ATTACH 'datatable://main' AS pg; + CREATE TABLE IF NOT EXISTS pg.exciting_809 AS + SELECT * FROM pg.fx_rates; + "#; + let assets = parse_assets(input).unwrap().assets; + assert!( + assets.iter().any(|a| a.path == "main/fx_rates" + && a.kind == AssetKind::DataTable + && a.access_type == Some(R)), + "expected read of main/fx_rates, got {:?}", + assets + ); + assert!( + assets.iter().any(|a| a.path == "main/exciting_809" + && a.access_type == Some(W)), + "expected write of main/exciting_809, got {:?}", + assets + ); + } + + #[test] + fn test_create_view_collects_upstream_read() { + let input = r#" + ATTACH 'datatable://main' AS pg; + CREATE VIEW pg.v AS SELECT * FROM pg.fx_rates; + "#; + let assets = parse_assets(input).unwrap().assets; + assert!( + assets.iter().any(|a| a.path == "main/fx_rates" && a.access_type == Some(R)), + "expected read of main/fx_rates, got {:?}", + assets + ); + } +} diff --git a/backend/parsers/windmill-parser-ts-asset/src/lib.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs index 39302fb6fa..b7fe0fea5c 100644 --- a/backend/parsers/windmill-parser-ts-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -5,8 +5,8 @@ use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, ObjectLit, Prop, use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; use swc_ecma_visit::{Visit, VisitWith}; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, SqlQueryDetails, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, SqlQueryDetails, }; use AssetUsageAccessType::*; @@ -38,10 +38,12 @@ pub fn parse_assets(code: &str) -> anyhow::Result { let mut assets_finder = AssetsFinder { assets: vec![], sql_queries: vec![], var_identifiers: HashMap::new() }; assets_finder.visit_module_items(&ast); - Ok(ParseAssetsOutput { - assets: merge_assets(assets_finder.assets), - sql_queries: assets_finder.sql_queries, - }) + let pipeline = parse_pipeline_annotations(code); + Ok(ParseAssetsOutput::new( + merge_assets(assets_finder.assets), + assets_finder.sql_queries, + pipeline, + )) } type VarAssetName = String; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 8f646c0503..b5d69db97b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -174,11 +174,10 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ast_node" -version = "0.9.9" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9184f2b369b3e8625712493c89b785881f27eedc6cde480a81883cef78868b2" +checksum = "0a184645bcc6f52d69d8e7639720699c6a99efb711f886e251ed1d16db8dd90e" dependencies = [ - "proc-macro2", "quote", "swc_macros_common", "syn 2.0.117", @@ -401,9 +400,9 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "better_scoped_tls" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297b153aa5e573b5863108a6ddc9d5c968bd0b20e75cc614ee9821d2f45679c7" +checksum = "7cd228125315b132eed175bf47619ac79b945b26e56b848ba203ae4ea8603609" dependencies = [ "scoped-tls", ] @@ -563,9 +562,6 @@ name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -dependencies = [ - "allocator-api2", -] [[package]] name = "byte-unit" @@ -613,6 +609,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-str" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2" +dependencies = [ + "bytes", + "serde", +] + [[package]] name = "bytesize" version = "1.3.3" @@ -1372,11 +1378,10 @@ dependencies = [ [[package]] name = "from_variant" -version = "0.1.9" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" +checksum = "308530a56b099da144ebc5d8e179f343ad928fa2b3558d1eb3db9af18d6eff43" dependencies = [ - "proc-macro2", "swc_macros_common", "syn 2.0.117", ] @@ -1751,15 +1756,14 @@ dependencies = [ [[package]] name = "hstr" -version = "0.2.17" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a26def229ea95a8709dad32868d975d0dd40235bd2ce82920e4a8fe692b5e0" +checksum = "31f11d91d7befd2ffd9d216e9e5ea1fae6174b20a2a1b67a688138003d2f4122" dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "phf 0.11.3", - "rustc-hash 1.1.0", + "rustc-hash 2.1.1", "triomphe", ] @@ -4136,10 +4140,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] -name = "serde" -version = "1.0.220" +name = "seq-macro" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceecad4c782e936ac90ecfd6b56532322e3262b14320abf30ce89a92ffdbfe22" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -4158,18 +4168,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.220" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddba47394f3b862d6ff6efdbd26ca4673e3566a307880a0ffb98f274bbe0ec32" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.220" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e1f3b1761e96def5ec6d04a6e7421c0404fa3cf5c0155f1e2848fae3d8cc08" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -4692,11 +4702,10 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "string_enum" -version = "0.4.4" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e383308aebc257e7d7920224fa055c632478d92744eca77f99be8fa1545b90" +checksum = "ae36a4951ca7bd1cfd991c241584a9824a70f6aff1e7d4f693fb3f2465e4030e" dependencies = [ - "proc-macro2", "quote", "swc_macros_common", "syn 2.0.117", @@ -4798,49 +4807,35 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" -[[package]] -name = "swc_allocator" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76aa0eb65c0f39f9b6d82a7e5192c30f7ac9a78f084a21f270de1d8c600ca388" -dependencies = [ - "bumpalo", - "hashbrown 0.14.5", - "ptr_meta", - "rustc-hash 1.1.0", - "triomphe", -] - [[package]] name = "swc_atoms" -version = "0.6.7" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6567e4e67485b3e7662b486f1565bdae54bd5b9d6b16b2ba1a9babb1e42125" +checksum = "3500dcf04c84606b38464561edc5e46f5132201cb3e23cf9613ed4033d6b1bb2" dependencies = [ "hstr", "once_cell", - "rustc-hash 1.1.0", "serde", ] [[package]] name = "swc_common" -version = "0.37.5" +version = "14.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" +checksum = "c2bb772b3a26b8b71d4e8c112ced5b5867be2266364b58517407a270328a2696" dependencies = [ + "anyhow", "ast_node", "better_scoped_tls", - "cfg-if", + "bytes-str", "either", "from_variant", "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 1.1.0", + "rustc-hash 2.1.1", "serde", "siphasher 0.3.11", - "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", "swc_visit", @@ -4851,32 +4846,36 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.118.2" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df" +checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ "bitflags", "is-macro", "num-bigint", + "once_cell", "phf 0.11.3", - "scoped-tls", + "rustc-hash 2.1.1", "string_enum", "swc_atoms", "swc_common", + "swc_visit", "unicode-id-start", ] [[package]] -name = "swc_ecma_parser" -version = "0.149.1" +name = "swc_ecma_lexer" +version = "23.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683dada14722714588b56481399c699378b35b2ba4deb5c4db2fb627a97fb54b" +checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ + "arrayvec", + "bitflags", "either", - "new_debug_unreachable", "num-bigint", - "num-traits", "phf 0.11.3", + "rustc-hash 2.1.1", + "seq-macro", "serde", "smallvec", "smartstring", @@ -4885,14 +4884,29 @@ dependencies = [ "swc_common", "swc_ecma_ast", "tracing", - "typed-arena", +] + +[[package]] +name = "swc_ecma_parser" +version = "24.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9011783c975ba592ffc09cd208ced92b1dfabb2e5e0ef453559e2e25286127" +dependencies = [ + "either", + "num-bigint", + "serde", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_lexer", + "tracing", ] [[package]] name = "swc_ecma_visit" -version = "0.104.8" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1c6802e68e51f336e8bc9644e9ff9da75d7da9c1a6247d532f2e908aa33e81" +checksum = "75a579aa8f9e212af521588df720ccead079c09fe5c8f61007cf724324aed3a0" dependencies = [ "new_debug_unreachable", "num-bigint", @@ -4905,9 +4919,9 @@ dependencies = [ [[package]] name = "swc_eq_ignore_macros" -version = "0.1.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" +checksum = "c16ce73424a6316e95e09065ba6a207eba7765496fed113702278b7711d4b632" dependencies = [ "proc-macro2", "quote", @@ -4916,9 +4930,9 @@ dependencies = [ [[package]] name = "swc_macros_common" -version = "0.3.14" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e18fbfe83811ffae2bb23727e45829a0d19c6870bced7c0f545cc99ad248dd" +checksum = "aae1efbaa74943dc5ad2a2fb16cbd78b77d7e4d63188f3c5b4df2b4dcd2faaae" dependencies = [ "proc-macro2", "quote", @@ -4927,9 +4941,9 @@ dependencies = [ [[package]] name = "swc_visit" -version = "0.6.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ceb044142ba2719ef9eb3b6b454fce61ab849eb696c34d190f04651955c613d" +checksum = "62fb71484b486c185e34d2172f0eabe7f4722742aad700f426a494bb2de232a2" dependencies = [ "either", "new_debug_unreachable", @@ -5065,7 +5079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -5595,12 +5609,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typeid" version = "1.0.3" @@ -6183,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.727.0" +version = "1.734.0" dependencies = [ "aho-corasick", "anyhow", @@ -6205,6 +6213,7 @@ dependencies = [ "croner", "dashmap", "equivalent", + "erased-serde", "futures", "futures-core", "gethostname", @@ -6263,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.727.0" +version = "1.734.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.727.0" +version = "1.734.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.727.0" +version = "1.734.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 3039b52af4..5e38fac8cf 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.727.0" +version = "1.734.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs index 7e67d563ff..4dde12ad8e 100644 --- a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -1,5 +1,6 @@ use windmill_parser::asset_parser::{ - merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, + merge_assets, parse_pipeline_annotations, AssetKind, AssetUsageAccessType, ParseAssetsOutput, + ParseAssetsResult, }; use crate::{parse_ansible_reqs, ResourceOrVariablePath}; @@ -39,5 +40,10 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(assets), + Vec::new(), + pipeline, + )) } diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 8a1f1bc097..5d9945cadd 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -50,7 +50,7 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result anyhow::Result anyhow::Result anyhow::Result,,...`). A newline or other config-meaningful +/// character would let a script inject arbitrary `[defaults]` directives (e.g. +/// `library`, `action_plugins`) and execute attacker-controlled code on the worker, +/// and a `,` would smuggle in an extra entry. Restrict entries to the `label@source` +/// charset so neither is possible. +pub fn validate_vault_id(value: &str) -> anyhow::Result<()> { + let is_valid = !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-' | '/' | '@')); + if !is_valid { + return Err(anyhow!( + "Invalid vault_id `{value}`: expected `label@filename` using only letters, digits and the characters `.`, `_`, `-`, `/`, `@`" + )); + } + Ok(()) +} + pub fn parse_ansible_reqs( inner_content: &str, ) -> anyhow::Result<(String, Option, String)> { @@ -528,6 +547,7 @@ pub fn parse_ansible_reqs( let Yaml::String(filename) = f else { return Err(anyhow!("The elements of the vault_id field should be strings in the format: `label@filename`")); }; + validate_vault_id(filename)?; ret.vault_id.push(filename.to_string()); } } @@ -1051,4 +1071,55 @@ delegate_to_git_repo: Some("inventories/{{ env }}") ); } + + #[test] + fn test_parse_vault_id_valid() { + let p = r#" +--- +vault_id: + - dev@vault_pass_dev.txt + - prod@./secrets/prod-pass +--- +- name: Test + hosts: all +"#; + let (_, reqs, _) = parse_ansible_reqs(p).unwrap(); + assert_eq!( + reqs.unwrap().vault_id, + vec![ + "dev@vault_pass_dev.txt".to_string(), + "prod@./secrets/prod-pass".to_string() + ] + ); + } + + #[test] + fn test_parse_vault_id_rejects_newline_injection() { + let p = "---\nvault_id:\n - \"default@/tmp/wm/x\\nlibrary = /tmp/wm/evil_modules\"\n---\n- name: Test\n hosts: all\n"; + assert!(parse_ansible_reqs(p).is_err()); + } + + #[test] + fn test_parse_vault_id_rejects_comma() { + let p = r#" +--- +vault_id: + - "a@b,c@d" +--- +- name: Test + hosts: all +"#; + assert!(parse_ansible_reqs(p).is_err()); + } + + #[test] + fn test_validate_vault_id() { + assert!(validate_vault_id("default@/tmp/wm/pass").is_ok()); + assert!(validate_vault_id("dev@pass.txt").is_ok()); + assert!(validate_vault_id("").is_err()); + assert!(validate_vault_id("a@b\nlibrary = /evil").is_err()); + assert!(validate_vault_id("a@b,c@d").is_err()); + assert!(validate_vault_id("a@b c").is_err()); + assert!(validate_vault_id("a@b=c").is_err()); + } } diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4239c1854c..61e30a343f 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,6 +1,15 @@ use serde::Serialize; use std::collections::BTreeMap; +// Token recognized inside declared asset URIs that the runtime substitutes +// with the current partition value (e.g. `s3://lake/{partition}.parquet` +// becomes `s3://lake/2024-04-29.parquet` when materialized for that day). +// Substitution is the parser's job only at lineage/graph time — at runtime +// the pipeline worker performs the actual replacement before emitting +// asset signals. Kept as a single well-known token so the parser never +// has to guess which `{...}` placeholders are partition variables. +pub const PARTITION_TOKEN: &str = "{partition}"; + #[derive(Serialize, PartialEq, Clone, Copy, Debug)] #[serde(rename_all(serialize = "lowercase"))] pub enum AssetUsageAccessType { @@ -47,6 +56,242 @@ pub struct SqlQueryDetails { pub struct ParseAssetsOutput { pub assets: Vec, pub sql_queries: Vec, + // Bare `// pipeline` (or `#` / `--`) on its own line — opt-in marker + // that sets auto_kind='pipeline' and includes the script in its + // folder's pipeline. Pipeline membership is broader than + // materialization: scripts that only assert/notify/clean up are + // members too. Outputs (when present) come from parser-detected + // `w`/`rw` usages in `assets`, not from this marker. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub in_pipeline: bool, + // Trigger annotations — execution DAG edges. Each is an independent OR + // (any fires the script). Empty = script has no automatic triggers + // (still runnable manually / via existing cron triggers). Includes both + // top-level `// schedule "..."` and `// on ...` forms. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub triggers: Vec, + // `// partitioned [opts]` — declares that this pipeline script + // produces partitioned output. The runtime resolves `{partition}` in + // declared output URIs to the current partition value before signaling + // downstream consumers. At most one per script. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub partition: Option, + // `// freshness ` — SLA stating outputs must be at most + // `duration` old. Active backstop: when no other trigger has fired the + // script within the window, a watchdog re-runs it. Distinct from + // schedule (which is producer cadence); freshness is consumer SLA and + // applies regardless of which trigger last fired. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub freshness: Option, + // `// trigger all` → AND join barrier; default (`any`) = OR (current + // behaviour). Threaded to the deploy path which persists it on the + // subscriber's trigger rows. + #[serde(skip_serializing_if = "JoinMode::is_any", default)] + pub join_mode: JoinMode, + // `// debounce ` — script-level default debounce window for this + // script's asset inputs. A per-`// on … debounce=` overrides it. Raw + // duration string, parsed to seconds at deploy (parser-light, like + // freshness). Absent = no debounce (fan-out, current behaviour). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub debounce_default: Option, + // `// tag ` — overrides the script's worker tag at deploy. Source + // wins over any UI-set value, matching the wipe-and-reinsert convention + // of other pipeline annotations (`// schedule`, `// on`). Absent = keep + // whatever the caller (UI / CLI) supplied. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub tag: Option, + // `// retry []` — re-run a pipeline-cascade triggered + // script up to `count` times on failure, waiting `delay` between + // attempts. Applies only to runs launched via the asset/schedule + // cascade (the rows in `script_trigger`); manual UI runs are unaffected. + // The delay is a raw duration string parsed at deploy (parser-light). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub retry: Option, + // `// materialize [manual] [append] [key=]` — + // managed-materialization target + its strategy. At most one per script. + // Drives the worker's write-strategy + snapshot capture. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub materialize: Option, +} + +#[derive(Serialize, Debug, PartialEq, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum TriggerSpec { + // Refresh when `` changes. Kind comes from parse_asset_syntax so + // it matches the `asset` table. `debounce` is the optional per-input + // `// on … debounce=` override (raw duration string); it takes + // precedence over the script-level `// debounce` default, resolved at + // deploy. + Asset { + asset_kind: AssetKind, + path: String, + #[serde(skip_serializing_if = "Option::is_none", default)] + debounce: Option, + }, + // `// on ` — marker-only declaration that this script wants to be + // triggered by a native trigger of the given kind. No path: the binding + // is the trigger row's own `script_path` field (set when the user creates + // the kafka/mqtt/schedule/… trigger in its dedicated UI). The graph + // endpoint discovers attached triggers by `WHERE script_path = ` and surfaces a "missing" placeholder when an annotation has no + // matching row. + Schedule, + Webhook, + Email, + Kafka, + Mqtt, + Nats, + Postgres, + Sqs, + Gcp, + // `// on data_upload` — UI-first entry point. Unlike the other native + // kinds there is no external event source and no trigger row anywhere: + // the script declares an `S3Object` input parameter and the user uploads + // a file via the auto-generated S3 picker, which runs the pipeline. The + // graph renders it as a clickable upload source (never a "missing" + // placeholder, mirroring webhook). + #[serde(rename = "data_upload")] + DataUpload, +} + +impl TriggerSpec { + // A `// on ` whose declared path contains the `{partition}` + // token is *partition-bearing*: in an AND join its concrete partition + // value is the join key. Non-asset triggers and assets without the + // token are reference/presence-only inputs that never define the + // partition (the case-3 guard). + pub fn is_partition_bearing(&self) -> bool { + matches!(self, TriggerSpec::Asset { path, .. } if path.contains(PARTITION_TOKEN)) + } +} + +// Partitioning declaration for a pipeline script. `daily`/`hourly`/`weekly`/ +// `monthly` are time-based with the runtime supplying the current +// partition value derived from the trigger context (schedule fire time, +// freshness window, manual run arg). `dynamic` extracts the value from the +// triggering payload via JSONPath — used for per-tenant / per-shard / +// per-event-id pipelines where the partition key isn't a wall-clock value. +#[derive(Serialize, Debug, PartialEq, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum PartitionKind { + Daily, + Hourly, + Weekly, + Monthly, + Dynamic { key: String }, +} + +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct PartitionSpec { + #[serde(flatten)] + pub kind: PartitionKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub tz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + // ISO-8601 (e.g. "2024-01-01") or kind-specific start anchor. Older + // partitions before this anchor are not backfilled. + #[serde(skip_serializing_if = "Option::is_none")] + pub start: Option, +} + +// Freshness SLA. The duration is kept as a raw string ("1h", "30m", "2d") +// and validated downstream — the parser deliberately doesn't bind to a +// specific duration crate so the annotation grammar stays parser-light. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct FreshnessSpec { + pub duration: String, +} + +// Retry policy declared via `// retry []`. The delay is kept +// as a raw duration string (mirrors freshness/debounce_default) and resolved +// to seconds at deploy via `parse_duration_secs`; absent = back-to-back +// re-runs with no inter-attempt wait. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct RetrySpec { + pub count: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub delay: Option, +} + +// `// materialize [manual] [append] [key=]` — declares that this +// script produces a *managed* materialization of `` (a `ducklake://` +// table). By default the runtime generates the write DDL around the script's +// single trailing `SELECT` and owns idempotency, partition-state and snapshot +// capture. `manual` is the escape hatch: the script writes its own DDL and the +// runtime only records state (track-only). The reconciliation strategy options +// (`append`, `key=`) apply to managed mode: none → DELETE-by-partition + +// INSERT (replace); `key=` → MERGE (dedup within slice); `append` → +// INSERT-only. `append` wins if both are given (deploy-time warning). +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct MaterializeSpec { + pub target_kind: AssetKind, + pub target_path: String, + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub manual: bool, + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub append: bool, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub unique_key: Option, +} + +// `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger +// firing runs the script (current behaviour). `All` = AND: the script +// runs only once every partition-bearing input has materialized at the +// same partition (plus every reference input exists) — the join barrier. +#[derive(Serialize, Debug, PartialEq, Eq, Clone, Copy, Default)] +#[serde(rename_all = "lowercase")] +pub enum JoinMode { + #[default] + Any, + All, +} + +impl JoinMode { + pub fn is_any(&self) -> bool { + matches!(self, JoinMode::Any) + } +} + +// All pipeline-level annotations parsed off a script's source. Returned by +// `parse_pipeline_annotations` and forwarded into `ParseAssetsOutput`. +#[derive(Default, Debug, PartialEq, Clone)] +pub struct PipelineAnnotations { + pub in_pipeline: bool, + pub triggers: Vec, + pub partition: Option, + pub freshness: Option, + pub join_mode: JoinMode, + pub debounce_default: Option, + pub tag: Option, + pub retry: Option, + pub materialize: Option, +} + +impl ParseAssetsOutput { + /// Build from detected assets/queries plus the script's parsed + /// pipeline annotations, so each language asset-parser does not + /// re-list the per-annotation fields (one call site instead of six + /// lines kept in lockstep across the parser crates). + pub fn new( + assets: Vec, + sql_queries: Vec, + pipeline: PipelineAnnotations, + ) -> Self { + ParseAssetsOutput { + assets, + sql_queries, + in_pipeline: pipeline.in_pipeline, + triggers: pipeline.triggers, + partition: pipeline.partition, + freshness: pipeline.freshness, + join_mode: pipeline.join_mode, + debounce_default: pipeline.debounce_default, + tag: pipeline.tag, + retry: pipeline.retry, + materialize: pipeline.materialize, + } + } } #[derive(Debug, Clone, Serialize)] @@ -64,13 +309,12 @@ pub fn merge_assets(assets: Vec) -> Vec { .iter_mut() .find(|x| x.path == asset.path && x.kind == asset.kind) { - // merge access types + // merge access types — a None on either side means ambiguous + // usage (unknown access), which poisons the merge to None; + // otherwise delegate to the shared truth table. existing.access_type = match (asset.access_type, existing.access_type) { (None, _) | (_, None) => None, - (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), - (Some(RW), _) | (_, Some(RW)) => Some(RW), - (Some(R), Some(R)) => Some(R), - (Some(W), Some(W)) => Some(W), + (Some(a), Some(b)) => Some(merge_access_types(a, b)), }; // merge columns: union the column sets and merge access types per column existing.columns = merge_column_maps(existing.columns.take(), asset.columns); @@ -153,3 +397,847 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("datatable://", AssetKind::DataTable), ("volume://", AssetKind::Volume), ]; + +// Tokenize a `key=value [key="quoted value"] ...` option string. Bare +// values run until the next whitespace; quoted values consume until the +// matching quote. Malformed pairs (missing `=` or empty key) are skipped +// rather than aborting the whole annotation. +// Split a `// on` right-hand side into the trigger ref and any trailing +// `key=value` opts. The opts section starts at the first whitespace token +// shaped like `=…` (e.g. `debounce=60s`); everything before is the +// asset/kind ref. Asset refs aren't expected to contain a space then an +// `ident=` token — the same assumption `// partitioned` already makes. +fn split_trailing_kv_opts(s: &str) -> (&str, BTreeMap) { + let mut split_at: Option = None; + for tok in s.split_whitespace() { + // `split_whitespace` yields slices borrowed from `s`, so the exact + // byte offset is the pointer delta — substring search (`find`) would + // misfire when an earlier token also appears inside a later one. + let tok_start = tok.as_ptr() as usize - s.as_ptr() as usize; + if let Some(eq) = tok.find('=') { + let key = &tok[..eq]; + if !key.is_empty() + && key.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') + && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + { + split_at = Some(tok_start); + break; + } + } + } + match split_at { + Some(i) => (s[..i].trim_end(), parse_kv_opts(&s[i..])), + None => (s.trim_end(), BTreeMap::new()), + } +} + +fn parse_kv_opts(s: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut chars = s.chars().peekable(); + loop { + while chars.peek().map_or(false, |c| c.is_whitespace()) { + chars.next(); + } + if chars.peek().is_none() { + break; + } + let mut key = String::new(); + while let Some(&c) = chars.peek() { + if c == '=' || c.is_whitespace() { + break; + } + key.push(c); + chars.next(); + } + if chars.peek() != Some(&'=') || key.is_empty() { + // Malformed — skip until next whitespace to recover. + while chars.peek().map_or(false, |c| !c.is_whitespace()) { + chars.next(); + } + continue; + } + chars.next(); // consume '=' + let value = match chars.peek().copied() { + Some(q @ ('"' | '\'')) => { + chars.next(); + let mut v = String::new(); + while let Some(&c) = chars.peek() { + chars.next(); + if c == q { + break; + } + v.push(c); + } + v + } + _ => { + let mut v = String::new(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + break; + } + v.push(c); + chars.next(); + } + v + } + }; + out.insert(key, value); + } + out +} + +// Scan raw source for pipeline annotations. Language-agnostic: any line +// whose first non-whitespace tokens are a comment prefix (`//`, `#`, or +// `--`) followed by one of the recognized keywords: +// - `pipeline` → opt-in marker (must be alone on the line) +// - `on ` → asset / native trigger edge (including +// the marker-only `on schedule` form) +// - `partitioned [opts]` → partition declaration +// - `freshness ` → SLA / active backstop +// - `tag ` → worker-tag override (annotation wins +// over UI-set value at deploy) +// - `retry []` → cascade-only retry policy +// +// `// pipeline` is intentionally strict — only whitespace allowed after the +// keyword. Without that constraint, casual prose like `// pipeline broken +// on staging` would false-positive (the word "pipeline" is far more common +// in normal comments than "materialize" was). +// +// `partition`, `freshness`, `tag`, and `retry` use first-write-wins; if +// multiple lines declare them, the first one is kept (last would be +// reasonable too, but first matches the file-top convention developers +// follow). +// Try to consume `` as a complete word from `rest`. Returns the trailing +// text after the keyword if it matched (empty or whitespace-bounded), +// `None` otherwise. Prevents `partitioned` matching `partition`, `pipelines` +// matching `pipeline`, etc. Mirrors `consumeKeyword` in +// parsePipelineAnnotations.ts. +fn consume_keyword<'a>(rest: &'a str, kw: &str) -> Option<&'a str> { + let after = rest.strip_prefix(kw)?; + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + Some(after) + } else { + None + } +} + +pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { + let mut out = PipelineAnnotations::default(); + + for raw_line in code.lines() { + let line = raw_line.trim_start(); + let rest = if let Some(r) = line.strip_prefix("//") { + r + } else if let Some(r) = line.strip_prefix("--") { + r + } else if let Some(r) = line.strip_prefix('#') { + r + } else { + continue; + }; + let rest = rest.trim_start(); + + if let Some(after_kw) = consume_keyword(rest, "pipeline") { + // Strict: keyword must be the only content on the line. Rejects + // `pipeline broken`, `pipelines`, `pipeline-related`, etc. + if after_kw.trim().is_empty() { + out.in_pipeline = true; + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "partitioned") { + if out.partition.is_none() { + if let Some(spec) = parse_partitioned_spec(after_kw.trim()) { + out.partition = Some(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "freshness") { + let dur = after_kw.trim(); + if !dur.is_empty() && out.freshness.is_none() { + out.freshness = Some(FreshnessSpec { duration: dur.to_string() }); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "trigger") { + match after_kw.trim() { + "all" => out.join_mode = JoinMode::All, + "any" => out.join_mode = JoinMode::Any, + // Unknown value — leave the default rather than guess. + _ => {} + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "debounce") { + let dur = after_kw.trim(); + if !dur.is_empty() && out.debounce_default.is_none() { + out.debounce_default = Some(dur.to_string()); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "tag") { + let name = after_kw.trim(); + if !name.is_empty() && out.tag.is_none() { + out.tag = Some(name.to_string()); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "retry") { + if out.retry.is_none() { + if let Some(spec) = parse_retry_spec(after_kw.trim()) { + out.retry = Some(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "materialize") { + if out.materialize.is_none() { + if let Some(spec) = parse_materialize_spec(after_kw.trim()) { + out.materialize = Some(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "on") { + let spec_text = after_kw.trim(); + if spec_text.is_empty() { + continue; + } + // Split off trailing `key=value` opts (e.g. `debounce=60s`) + // from the asset/kind ref. The per-input debounce override is + // only meaningful for asset inputs (cascade fan-out); other + // trigger kinds ignore it. + let (ref_part, opts) = split_trailing_kv_opts(spec_text); + if let Some(mut trig) = parse_trigger_spec(ref_part) { + if let TriggerSpec::Asset { debounce, .. } = &mut trig { + *debounce = opts + .get("debounce") + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + } + if !out.triggers.contains(&trig) { + out.triggers.push(trig); + } + } + } + } + + out +} + +// Parse a `// retry []` right-hand side. `` is a +// non-negative decimal; `` is an optional raw duration string left +// for `parse_duration_secs` to validate at deploy. A bare zero count (or +// non-numeric token) is rejected — that is, the annotation must encode a +// real retry policy or be omitted, so a typo (`// retry retry 3`) fails +// safe rather than silently disabling cascade retries. +fn parse_retry_spec(s: &str) -> Option { + let mut split = s.splitn(2, char::is_whitespace); + let count_word = split.next()?.trim(); + let count: u32 = count_word.parse().ok()?; + if count == 0 { + return None; + } + let delay = split + .next() + .map(|d| d.trim().to_string()) + .filter(|d| !d.is_empty()); + Some(RetrySpec { count, delay }) +} + +// Parse a `// materialize [manual] [append] [key=]` right-hand +// side. An optional leading `manual` token (whitespace-delimited) opts out of +// managed mode (track-only). The next whitespace token is the target asset URI +// (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the +// remainder are strategy options — bare `append` and `key=` (merge key), +// which apply to managed mode only. A missing/empty target yields `None` (the +// annotation is dropped, fail-safe). +fn parse_materialize_spec(s: &str) -> Option { + let (manual, rest) = match s.strip_prefix("manual") { + Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => { + (true, after.trim_start()) + } + _ => (false, s), + }; + let mut it = rest.trim().splitn(2, char::is_whitespace); + let asset_tok = it.next()?; + let opts_str = it.next().unwrap_or(""); + let (target_kind, path) = parse_asset_syntax(asset_tok.trim(), true)?; + if path.is_empty() { + return None; + } + let append = opts_str.split_whitespace().any(|t| t == "append"); + let unique_key = parse_kv_opts(opts_str) + .get("key") + .filter(|k| !k.is_empty()) + .cloned(); + Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key }) +} + +// Parse a `// partitioned [opts]` right-hand side. Recognized kinds: +// `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start), +// and `dynamic key=""` (plus optional format). +fn parse_partitioned_spec(s: &str) -> Option { + let mut split = s.splitn(2, char::is_whitespace); + let kind_word = split.next()?; + let opts_str = split.next().unwrap_or(""); + let opts = parse_kv_opts(opts_str); + let kind = match kind_word { + "daily" => PartitionKind::Daily, + "hourly" => PartitionKind::Hourly, + "weekly" => PartitionKind::Weekly, + "monthly" => PartitionKind::Monthly, + "dynamic" => { + let key = opts.get("key")?.clone(); + if key.is_empty() { + return None; + } + PartitionKind::Dynamic { key } + } + _ => return None, + }; + Some(PartitionSpec { + kind, + tz: opts.get("tz").cloned(), + format: opts.get("format").cloned(), + start: opts.get("start").cloned(), + }) +} + +// Parse a single `on ` right-hand side. Accepted forms: +// — where is one of +// webhook | email | kafka | mqtt | nats | postgres | sqs | gcp +// (e.g. s3://bucket/key, $res:f/foo) +// +// Native trigger keywords are *marker-only* — no trailing path. The actual +// binding lives on the native trigger row (`script_path` column). Anything +// trailing the keyword is rejected so the form stays unambiguous. +fn parse_trigger_spec(s: &str) -> Option { + // Marker-only native trigger keywords. The match table keeps the + // annotation set in lockstep with `TriggerSpec`. `schedule` is in here + // too — the cron lives on the schedule row the user creates separately; + // the annotation is just the binding declaration. + const NATIVE_KINDS: &[(&str, TriggerSpec)] = &[ + ("schedule", TriggerSpec::Schedule), + ("webhook", TriggerSpec::Webhook), + ("email", TriggerSpec::Email), + ("kafka", TriggerSpec::Kafka), + ("mqtt", TriggerSpec::Mqtt), + ("nats", TriggerSpec::Nats), + ("postgres", TriggerSpec::Postgres), + ("sqs", TriggerSpec::Sqs), + ("gcp", TriggerSpec::Gcp), + ("data_upload", TriggerSpec::DataUpload), + ]; + for (kw, spec) in NATIVE_KINDS { + if let Some(rest) = s.strip_prefix(kw) { + // Must be a complete word — `kafkalike` doesn't match `kafka`. + // Trailing whitespace alone is fine; any non-empty trailing + // content is treated as malformed (the annotation is marker-only). + if !rest.is_empty() && !rest.starts_with(|c: char| c.is_whitespace()) { + continue; + } + if !rest.trim().is_empty() { + return None; + } + return Some(spec.clone()); + } + } + + let (kind, path) = parse_asset_syntax(s, false)?; + // `debounce` is attached by the caller from the `// on` line's opts. + Some(TriggerSpec::Asset { asset_kind: kind, path: path.to_string(), debounce: None }) +} + +#[cfg(test)] +mod pipeline_annotation_tests { + use super::*; + + #[test] + fn bare_pipeline_marker() { + let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')"); + assert!(out.in_pipeline); + assert!(out.triggers.is_empty()); + } + + #[test] + fn pipeline_marker_strict_grammar_rejects_trailing_words() { + // Strict — the word "pipeline" is common in casual prose, so trailing + // content disqualifies the line. Trailing whitespace alone is fine. + let out = parse_pipeline_annotations( + "// pipeline broken on staging\n# pipeline this through grep\n-- pipeline_v2", + ); + assert!(!out.in_pipeline); + + let out = parse_pipeline_annotations("// pipeline \n"); + assert!(out.in_pipeline); + } + + #[test] + fn rejects_pipeline_keyword_variants() { + let out = parse_pipeline_annotations("// pipelines\n# pipelined\n-- pipeline-foo"); + assert!(!out.in_pipeline); + } + + #[test] + fn on_schedule_marker() { + // `// on schedule` is marker-only — the binding is the schedule row's + // own `script_path` field, just like kafka/mqtt/etc. + let out = parse_pipeline_annotations("// on schedule"); + assert_eq!(out.triggers.len(), 1); + assert_eq!(out.triggers[0], TriggerSpec::Schedule); + } + + #[test] + fn rejects_schedule_with_trailing_content() { + // Marker-only — the old `// schedule ""` form is gone, and a + // trailing path/cron on the `on schedule` form is malformed. + let out = parse_pipeline_annotations("// on schedule \"0 0 * * *\""); + assert!(out.triggers.is_empty()); + let out = parse_pipeline_annotations("// schedule \"0 0 * * *\""); + assert!(out.triggers.is_empty()); + } + + #[test] + fn on_asset_ts_py_sql() { + let code = "// on s3://a/b\n# on datatable://main\n-- on $res:f/foo"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 3); + assert!(matches!( + out.triggers[0], + TriggerSpec::Asset { asset_kind: AssetKind::S3Object, .. } + )); + assert!(matches!( + out.triggers[1], + TriggerSpec::Asset { asset_kind: AssetKind::DataTable, .. } + )); + assert!(matches!( + out.triggers[2], + TriggerSpec::Asset { asset_kind: AssetKind::Resource, .. } + )); + } + + #[test] + fn on_deduplicates() { + let code = "// on s3://a/b\n# on s3://a/b"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 1); + } + + #[test] + fn rejects_unknown_trigger_spec() { + let out = parse_pipeline_annotations("// on unknown://nope\n# schedule"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn join_mode_defaults_to_any() { + let out = parse_pipeline_annotations("// on s3://a/b"); + assert_eq!(out.join_mode, JoinMode::Any); + assert!(out.join_mode.is_any()); + } + + #[test] + fn trigger_all_sets_and_mode() { + let out = parse_pipeline_annotations( + "// pipeline\n// on s3://lake/{partition}/x\n// trigger all", + ); + assert_eq!(out.join_mode, JoinMode::All); + assert!(!out.join_mode.is_any()); + } + + #[test] + fn trigger_any_explicit_and_other_prefixes() { + // explicit `any`, plus `#` / `--` comment prefixes are accepted. + assert_eq!( + parse_pipeline_annotations("# trigger all\n-- trigger any").join_mode, + JoinMode::Any + ); + assert_eq!( + parse_pipeline_annotations("-- trigger all").join_mode, + JoinMode::All + ); + } + + #[test] + fn trigger_unknown_or_glued_keeps_default() { + // unknown value, and `triggerall` (no whitespace) must not match. + assert_eq!( + parse_pipeline_annotations("// trigger bogus").join_mode, + JoinMode::Any + ); + assert_eq!( + parse_pipeline_annotations("// triggerall").join_mode, + JoinMode::Any + ); + } + + #[test] + fn is_partition_bearing_only_for_tokened_assets() { + let out = parse_pipeline_annotations( + "// on s3://lake/raw/{partition}/events.parquet\n\ + // on s3://lake/dim/customers.parquet\n\ + // on schedule", + ); + assert_eq!(out.triggers.len(), 3); + assert!(out.triggers[0].is_partition_bearing()); + assert!(!out.triggers[1].is_partition_bearing()); + assert!(!out.triggers[2].is_partition_bearing()); + } + + fn asset_debounce(t: &TriggerSpec) -> Option<&str> { + match t { + TriggerSpec::Asset { debounce, .. } => debounce.as_deref(), + _ => None, + } + } + + #[test] + fn debounce_none_by_default() { + let out = parse_pipeline_annotations("// on s3://a/b"); + assert_eq!(out.debounce_default, None); + assert_eq!(asset_debounce(&out.triggers[0]), None); + } + + #[test] + fn script_level_debounce_default() { + let out = parse_pipeline_annotations("// debounce 30s\n// on s3://a/b"); + assert_eq!(out.debounce_default.as_deref(), Some("30s")); + // Per-edge override absent — precedence (edge ?? default) is + // resolved at deploy, so the Asset itself stays None here. + assert_eq!(asset_debounce(&out.triggers[0]), None); + } + + #[test] + fn on_level_debounce_override() { + let out = parse_pipeline_annotations( + "// debounce 30s\n\ + // on s3://lake/{partition}/raw.parquet debounce=60s\n\ + // on $res:f/cfg", + ); + assert_eq!(out.debounce_default.as_deref(), Some("30s")); + assert_eq!(out.triggers.len(), 2); + assert_eq!(asset_debounce(&out.triggers[0]), Some("60s")); + // ref still parses correctly with the trailing opt stripped. + assert!(out.triggers[0].is_partition_bearing()); + assert_eq!(asset_debounce(&out.triggers[1]), None); + } + + #[test] + fn debounce_keyword_strictness_and_first_wins() { + // `debounced` (no space) must not match; empty ignored; first wins. + let out = parse_pipeline_annotations( + "// debounced nope\n// debounce\n// debounce 1m\n// debounce 5m", + ); + assert_eq!(out.debounce_default.as_deref(), Some("1m")); + } + + #[test] + fn native_trigger_keywords_are_marker_only() { + // Marker form: `// on kafka` parses to the unit variant. + let out = parse_pipeline_annotations("// on kafka"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::Kafka)); + + // Old path-bearing form is rejected (no path on native markers). + let out = parse_pipeline_annotations("// on webhook f/foo"); + assert!(out.triggers.is_empty()); + + // Trailing key=value opts are silently dropped by the line-level + // KV splitter before parse_trigger_spec sees them — same behaviour + // for both asset and native kinds. The opts have no meaning for a + // marker, but the marker still parses. + let out = parse_pipeline_annotations("// on mqtt debounce=30s"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::Mqtt)); + + // `kafkalike` mustn't match `kafka`. + let out = parse_pipeline_annotations("// on kafkalike"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn all_native_marker_keywords_parse() { + let code = "// on webhook\n// on email\n// on kafka\n// on mqtt\n\ + // on nats\n// on postgres\n// on sqs\n// on gcp\n// on data_upload"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 9); + assert!(matches!(out.triggers[0], TriggerSpec::Webhook)); + assert!(matches!(out.triggers[1], TriggerSpec::Email)); + assert!(matches!(out.triggers[2], TriggerSpec::Kafka)); + assert!(matches!(out.triggers[3], TriggerSpec::Mqtt)); + assert!(matches!(out.triggers[4], TriggerSpec::Nats)); + assert!(matches!(out.triggers[5], TriggerSpec::Postgres)); + assert!(matches!(out.triggers[6], TriggerSpec::Sqs)); + assert!(matches!(out.triggers[7], TriggerSpec::Gcp)); + assert!(matches!(out.triggers[8], TriggerSpec::DataUpload)); + } + + #[test] + fn data_upload_marker_is_marker_only() { + // `// on data_upload` parses to the unit variant — no path. + let out = parse_pipeline_annotations("// on data_upload"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::DataUpload)); + + // Trailing content makes it malformed (marker-only). + let out = parse_pipeline_annotations("// on data_upload f/foo"); + assert!(out.triggers.is_empty()); + + // `data_uploadish` mustn't match `data_upload`. + let out = parse_pipeline_annotations("// on data_uploadish"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn partitioned_daily() { + let code = "// partitioned daily tz=\"UTC\" format=\"YYYY-MM-DD\" start=\"2024-01-01\""; + let out = parse_pipeline_annotations(code); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Daily); + assert_eq!(p.tz.as_deref(), Some("UTC")); + assert_eq!(p.format.as_deref(), Some("YYYY-MM-DD")); + assert_eq!(p.start.as_deref(), Some("2024-01-01")); + } + + #[test] + fn partitioned_hourly_minimal() { + let out = parse_pipeline_annotations("// partitioned hourly"); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Hourly); + assert!(p.tz.is_none()); + } + + #[test] + fn partitioned_dynamic_requires_key() { + let out = parse_pipeline_annotations("// partitioned dynamic"); + assert!(out.partition.is_none()); + let out = parse_pipeline_annotations("// partitioned dynamic key=\"$.tenant_id\""); + let p = out.partition.expect("partition"); + assert_eq!( + p.kind, + PartitionKind::Dynamic { key: "$.tenant_id".to_string() } + ); + } + + #[test] + fn partitioned_first_wins() { + let code = "// partitioned daily tz=\"UTC\"\n// partitioned hourly"; + let out = parse_pipeline_annotations(code); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Daily); + } + + #[test] + fn partitioned_unknown_kind_is_skipped() { + let out = parse_pipeline_annotations("// partitioned bogus"); + assert!(out.partition.is_none()); + } + + #[test] + fn freshness_basic() { + let out = parse_pipeline_annotations("// freshness 1h"); + assert_eq!(out.freshness.unwrap().duration, "1h"); + } + + #[test] + fn freshness_first_wins() { + let out = parse_pipeline_annotations("// freshness 1h\n# freshness 30m"); + assert_eq!(out.freshness.unwrap().duration, "1h"); + } + + #[test] + fn tag_basic() { + let out = parse_pipeline_annotations("// tag heavy"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + } + + #[test] + fn tag_first_wins() { + let out = parse_pipeline_annotations("// tag heavy\n# tag light"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + } + + #[test] + fn tag_empty_is_skipped() { + let out = parse_pipeline_annotations("// tag "); + assert!(out.tag.is_none()); + } + + #[test] + fn retry_count_only() { + let out = parse_pipeline_annotations("// retry 3"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay, None); + } + + #[test] + fn retry_with_delay() { + let out = parse_pipeline_annotations("// retry 3 5s"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + } + + #[test] + fn retry_first_wins() { + let out = parse_pipeline_annotations("// retry 3 5s\n# retry 1"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + } + + #[test] + fn retry_zero_is_skipped() { + let out = parse_pipeline_annotations("// retry 0 5s"); + assert!(out.retry.is_none()); + } + + #[test] + fn retry_non_numeric_count_is_skipped() { + let out = parse_pipeline_annotations("// retry many"); + assert!(out.retry.is_none()); + } + + #[test] + fn materialize_managed_default() { + let out = parse_pipeline_annotations("// materialize ducklake://analytics/orders_daily"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_kind, AssetKind::Ducklake); + assert_eq!(m.target_path, "analytics/orders_daily"); + // managed by default; replace strategy (no append / key) + assert!(!m.manual); + assert!(!m.append); + assert_eq!(m.unique_key, None); + } + + #[test] + fn materialize_manual_escape_hatch() { + let out = + parse_pipeline_annotations("// materialize manual ducklake://analytics/orders_daily"); + let m = out.materialize.expect("materialize"); + assert!(m.manual); + assert_eq!(m.target_path, "analytics/orders_daily"); + } + + #[test] + fn materialize_merge_and_append_options() { + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders_daily key=order_id"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.unique_key.as_deref(), Some("order_id")); + assert!(!m.append); + + let out = parse_pipeline_annotations("// materialize ducklake://a/events append"); + let m = out.materialize.expect("materialize"); + assert!(m.append); + assert_eq!(m.unique_key, None); + } + + #[test] + fn materialize_default_syntax_shorthand() { + let out = parse_pipeline_annotations("// materialize ducklake"); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_kind, AssetKind::Ducklake); + assert_eq!(m.target_path, "main"); + assert!(!m.manual); + } + + #[test] + fn materialize_manual_only_is_dropped() { + // `manual` with no target is not a valid materialization. + let out = parse_pipeline_annotations("// materialize manual"); + assert!(out.materialize.is_none()); + } + + #[test] + fn materialize_first_wins() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/x\n# materialize manual ducklake://b/y", + ); + let m = out.materialize.expect("materialize"); + assert_eq!(m.target_path, "a/x"); + assert!(!m.manual); + } + + #[test] + fn combined() { + let code = concat!( + "// pipeline\n", + "// on schedule\n", + "// on s3://in.csv\n", + "// partitioned daily tz=\"UTC\"\n", + "// freshness 2h\n", + "// tag heavy\n", + "// retry 3 5s\n", + "// materialize ducklake://analytics/orders_daily key=order_id\n" + ); + let out = parse_pipeline_annotations(code); + assert!(out.in_pipeline); + assert_eq!(out.triggers.len(), 2); + assert!(out.partition.is_some()); + assert_eq!(out.freshness.unwrap().duration, "2h"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + let m = out.materialize.expect("materialize"); + assert!(!m.manual); + assert_eq!(m.target_path, "analytics/orders_daily"); + assert_eq!(m.unique_key.as_deref(), Some("order_id")); + } + + #[test] + fn split_trailing_kv_opts_uses_exact_token_offset() { + // Regression for the token-offset computation in + // `split_trailing_kv_opts`. The asset ref's path token contains the + // exact text of the trailing `debounce=60s` opt as a substring, and + // the same `debounce=60s` text also appears a second time before the + // real opt. Offsetting by the `&str` slice's pointer is exact; a + // substring scan (`find`) is what this guards against regressing to. + let (ref_part, opts) = split_trailing_kv_opts("s3://lake/debounce=60s/raw debounce=60s"); + assert_eq!(ref_part, "s3://lake/debounce=60s/raw"); + assert_eq!(opts.get("debounce").map(String::as_str), Some("60s")); + + // End-to-end through the annotation parser: the ref must parse to the + // full S3 path (not truncated at the embedded `debounce=`), and the + // per-edge debounce override must be picked up from the trailing opt. + let out = parse_pipeline_annotations("// on s3://lake/debounce=60s/raw debounce=90s"); + assert_eq!(out.triggers.len(), 1); + match &out.triggers[0] { + TriggerSpec::Asset { asset_kind, path, debounce } => { + assert_eq!(*asset_kind, AssetKind::S3Object); + assert_eq!(path, "lake/debounce=60s/raw"); + assert_eq!(debounce.as_deref(), Some("90s")); + } + other => panic!("expected asset trigger, got {other:?}"), + } + } + + #[test] + fn kv_opts_quoted_with_spaces() { + let m = parse_kv_opts("a=\"hello world\" b=plain c='single quoted'"); + assert_eq!(m.get("a").unwrap(), "hello world"); + assert_eq!(m.get("b").unwrap(), "plain"); + assert_eq!(m.get("c").unwrap(), "single quoted"); + } + + #[test] + fn kv_opts_malformed_recovers() { + let m = parse_kv_opts("garbage a=ok =alone b=fine"); + assert_eq!(m.get("a").unwrap(), "ok"); + assert_eq!(m.get("b").unwrap(), "fine"); + assert!(m.get("garbage").is_none()); + } +} diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 5a7e90bf7e..19bc5602cd 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -13,6 +13,7 @@ use serde::Serialize; use serde_json::Value; pub mod asset_parser; +pub mod sql_materialize; /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) #[derive(Clone, Copy, Debug)] diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs new file mode 100644 index 0000000000..c1b0cc97aa --- /dev/null +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -0,0 +1,817 @@ +//! Eligibility classifier + materialization SQL codegen for managed `// materialize`. +//! +//! Managed `// materialize` (the default) promises the script is "setup +//! statements, then one trailing SELECT" — Windmill generates the write DDL +//! around that SELECT (the `// materialize manual` escape hatch opts out and +//! writes its own DDL). This module is the single source of truth for *which +//! block is that SELECT* and *what DDL gets generated*, so save-time validation +//! (deploy path) and run-time codegen (DuckDB executor) can never disagree. +//! +//! Everything here is pure and string-level: no SQL is executed, no type +//! inference is done. The classifier is leading-keyword based and deliberately +//! conservative — anything it can't positively recognize as a read-only output +//! or a known-safe setup statement is rejected, so a script is only accepted +//! for managed mode when its shape is unambiguous. + +/// One top-level statement's role in a wrap-mode script. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockClass { + /// Read-only relation the wrap writes from: `SELECT` / `WITH …SELECT` / + /// `FROM` (DuckDB from-first) / `VALUES` / `TABLE x` / `(UN)PIVOT`. + Output, + /// Known-safe preamble: `ATTACH` / `INSTALL` / `LOAD` / `SET` / `PRAGMA` / + /// `USE` / `CREATE TEMP …`. Runs verbatim before the generated write. + Setup, + /// Anything that writes or whose effect we can't vouch for: non-temp + /// `CREATE` / `INSERT` / `UPDATE` / `DELETE` / `MERGE` / `DROP` / `COPY` / + /// `ALTER` / `TRUNCATE`, or an unrecognized leading keyword. Disqualifies + /// managed mode (the user should use `// materialize manual`). + Disallowed, +} + +/// A script accepted for wrapping: zero+ setup blocks then one terminal SELECT. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WrapPlan { + /// Setup statements in source order, verbatim, **without** trailing `;`. + pub setup: Vec, + /// The single terminal output statement, verbatim, **without** trailing `;`. + pub output: String, +} + +/// Why a script is not eligible for managed `// materialize`. Carries enough to +/// render the targeted save-time messages. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WrapError { + /// No statements at all (empty / comments only). + Empty, + /// No terminal SELECT — nothing to wrap. + NoOutput, + /// More than one top-level SELECT. `count` is how many were found. + MultipleOutputs { count: usize }, + /// A SELECT exists but isn't the last statement (something runs after it). + OutputNotLast, + /// A write/unknown statement appears among the setup blocks. `snippet` is a + /// short prefix of the offending statement for the error message. + DisallowedBlock { snippet: String }, +} + +impl WrapError { + /// Human-facing, actionable message (matches the spec's rejection text). + pub fn message(&self) -> String { + let base = + "managed `// materialize` requires the script to be setup statements then a single trailing SELECT"; + let manual = "use `// materialize manual` to write the DDL yourself"; + match self { + WrapError::Empty => format!("{base}: the script is empty."), + WrapError::NoOutput => format!("{base}: found no SELECT — {manual}."), + WrapError::MultipleOutputs { count } => format!( + "{base}: found {count} SELECT statements; combine them with a CTE, or {manual}." + ), + WrapError::OutputNotLast => format!( + "{base}: found statements after the SELECT — move them above it, or {manual}." + ), + WrapError::DisallowedBlock { snippet } => { + format!("{base}: `{snippet}` writes or is unrecognized — {manual}.") + } + } + } +} + +/// Split SQL into top-level, `;`-separated statements, skipping line comments +/// (`-- …`), block comments (`/* … */`), single-quoted strings (`'…'` with +/// `''` escape) and double-quoted identifiers (`"…"`). Semicolons inside any of +/// those are not separators. Returns each statement trimmed, comments stripped, +/// empties dropped. Self-contained so the parser crate stays dependency-free; +/// it must stay behaviourally aligned with the executor's block splitter (both +/// route wrap through `classify_wrap`, so the split they see is this one). +pub fn split_statements(sql: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let bytes = sql.as_bytes(); + let mut i = 0; + let n = bytes.len(); + while i < n { + let c = bytes[i] as char; + // line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it + // is how Windmill pipeline annotations (`// materialize`, `// pipeline`, + // …) are written, and they sit above the SQL in the same script; strip + // them so they don't pollute the first statement block's classification + // or the generated setup SQL. + if (c == '-' && i + 1 < n && bytes[i + 1] == b'-') + || (c == '/' && i + 1 < n && bytes[i + 1] == b'/') + { + while i < n && bytes[i] != b'\n' { + i += 1; + } + continue; + } + // block comment + if c == '/' && i + 1 < n && bytes[i + 1] == b'*' { + i += 2; + while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + i += 1; + } + i += 2; + continue; + } + // single-quoted string + if c == '\'' { + cur.push(c); + i += 1; + while i < n { + cur.push(bytes[i] as char); + if bytes[i] == b'\'' { + // doubled '' is an escaped quote, stay in string + if i + 1 < n && bytes[i + 1] == b'\'' { + cur.push('\''); + i += 2; + continue; + } + i += 1; + break; + } + i += 1; + } + continue; + } + // double-quoted identifier + if c == '"' { + cur.push(c); + i += 1; + while i < n { + cur.push(bytes[i] as char); + if bytes[i] == b'"' { + i += 1; + break; + } + i += 1; + } + continue; + } + if c == ';' { + let t = cur.trim(); + if !t.is_empty() { + out.push(t.to_string()); + } + cur.clear(); + i += 1; + continue; + } + cur.push(c); + i += 1; + } + let t = cur.trim(); + if !t.is_empty() { + out.push(t.to_string()); + } + out +} + +/// Lowercased top-level keyword tokens of a single statement (parens collapsed +/// away: tokens *inside* balanced `(...)` are skipped, so a CTE body's verbs +/// don't leak up). Strings/identifiers are already gone from the split, but we +/// re-guard quotes defensively. Used to disambiguate `WITH …` and `CREATE …`. +fn top_level_keywords(stmt: &str) -> Vec { + let mut toks = Vec::new(); + let mut cur = String::new(); + let mut depth: i32 = 0; + let bytes = stmt.as_bytes(); + let mut i = 0; + let n = bytes.len(); + let flush = |cur: &mut String, toks: &mut Vec| { + if !cur.is_empty() { + toks.push(cur.to_lowercase()); + cur.clear(); + } + }; + while i < n { + let c = bytes[i] as char; + if c == '\'' || c == '"' { + let q = bytes[i]; + i += 1; + while i < n && bytes[i] != q { + i += 1; + } + i += 1; + continue; + } + if c == '(' { + flush(&mut cur, &mut toks); + depth += 1; + i += 1; + continue; + } + if c == ')' { + if depth > 0 { + depth -= 1; + } + i += 1; + continue; + } + if depth > 0 { + i += 1; + continue; + } + if c.is_alphanumeric() || c == '_' { + cur.push(c); + } else { + flush(&mut cur, &mut toks); + } + i += 1; + } + flush(&mut cur, &mut toks); + toks +} + +const OUTPUT_KW: &[&str] = &["select", "from", "values", "table", "pivot", "unpivot"]; +const SETUP_KW: &[&str] = &["attach", "install", "load", "set", "pragma", "use"]; +const WRITE_VERBS: &[&str] = &["insert", "update", "delete", "merge"]; + +/// Classify a single statement by its leading keyword (with `WITH`/`CREATE` +/// disambiguation). See [`BlockClass`]. +pub fn classify_block(stmt: &str) -> BlockClass { + let kws = top_level_keywords(stmt); + let Some(first) = kws.first().map(String::as_str) else { + return BlockClass::Disallowed; + }; + + // CREATE TEMP … is setup (staging); any other CREATE is a write. + if first == "create" { + let temp = kws + .iter() + .skip(1) + .take(3) + .any(|k| k == "temp" || k == "temporary"); + return if temp { + BlockClass::Setup + } else { + BlockClass::Disallowed + }; + } + + // WITH … : the main statement's verb decides. CTE bodies are parenthesized, + // so their verbs are not in `kws`; the first top-level write verb or SELECT + // after the CTE list is the real one. + if first == "with" { + for k in kws.iter().skip(1) { + if k == "select" { + return BlockClass::Output; + } + if WRITE_VERBS.contains(&k.as_str()) { + return BlockClass::Disallowed; + } + } + // `WITH x AS (...) SELECT` where SELECT got collapsed is impossible + // (SELECT here is top-level), so a WITH with no top-level verb is a + // malformed/unknown statement — reject conservatively. + return BlockClass::Disallowed; + } + + if OUTPUT_KW.contains(&first) { + return BlockClass::Output; + } + if SETUP_KW.contains(&first) { + return BlockClass::Setup; + } + BlockClass::Disallowed +} + +/// Validate a script for managed `// materialize` and, on success, return the +/// setup/output split. Enforces the four conditions from the spec: +/// 1. exactly one Output block, 2. it is last, 3. all preceding blocks are +/// Setup, 4. nothing after it. +pub fn classify_wrap(sql: &str) -> Result { + let stmts = split_statements(sql); + if stmts.is_empty() { + return Err(WrapError::Empty); + } + let classes: Vec = stmts.iter().map(|s| classify_block(s)).collect(); + + let output_idxs: Vec = classes + .iter() + .enumerate() + .filter(|(_, c)| **c == BlockClass::Output) + .map(|(i, _)| i) + .collect(); + + match output_idxs.len() { + 0 => return Err(WrapError::NoOutput), + 1 => {} + count => return Err(WrapError::MultipleOutputs { count }), + } + let out_idx = output_idxs[0]; + if out_idx != stmts.len() - 1 { + return Err(WrapError::OutputNotLast); + } + // Everything before the output must be Setup (no Disallowed preamble). + for (i, c) in classes.iter().enumerate().take(out_idx) { + if *c != BlockClass::Setup { + return Err(WrapError::DisallowedBlock { snippet: snippet(&stmts[i]) }); + } + } + Ok(WrapPlan { setup: stmts[..out_idx].to_vec(), output: stmts[out_idx].clone() }) +} + +fn snippet(stmt: &str) -> String { + let one_line: String = stmt.split_whitespace().collect::>().join(" "); + if one_line.chars().count() > 40 { + let truncated: String = one_line.chars().take(40).collect(); + format!("{truncated}…") + } else { + one_line + } +} + +// --------------------------------------------------------------------------- +// Codegen +// --------------------------------------------------------------------------- + +/// How a (partition of a) materialized table is reconciled on each run. +/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else +/// `unique_key` → `Merge`, else `Replace`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MaterializeStrategy { + /// DELETE the current partition, then INSERT — partition becomes exactly + /// what the SELECT returned. Full-refresh of the slice. + Replace, + /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows + /// absent from the SELECT are left in place. + Merge { unique_key: String }, + /// INSERT only — immutable event-log semantics. + Append, +} + +/// Inputs to materialization codegen, all resolved at run time by the worker. +/// Pure: produces SQL text; executes nothing. +#[derive(Debug, Clone)] +pub struct MaterializeCodegen<'a> { + /// Fully-qualified target, e.g. `_wm_target.orders_daily`. Always qualified + /// so a user `USE …;` in setup can't redirect the write. + pub target_qualified: &'a str, + /// The user's output SELECT (verbatim, no trailing `;`) — embedded as a + /// subquery so its own shape is irrelevant to the generated wrapper. + pub select_sql: &'a str, + /// Physical partition column added to the managed table. + pub partition_col: &'a str, + /// SQL expression for the current partition value — a literal like + /// `'2026-06-19'` or a bind placeholder. The caller is responsible for + /// safe quoting/binding. + pub partition_value_sql: &'a str, + /// Whether `// partitioned` applies. When false the table is unpartitioned + /// and the partition column / `SET PARTITIONED BY` are omitted. + pub partitioned: bool, + pub strategy: MaterializeStrategy, +} + +impl<'a> MaterializeCodegen<'a> { + /// The ordered statements that perform the materialization, to be run after + /// the setup blocks and inside the caller's execution. The first-run + /// bootstrap is idempotent (`IF NOT EXISTS`), so this is safe to run every + /// time. The DELETE/INSERT body is wrapped in one transaction so a partial + /// failure leaves the prior snapshot intact. Every strategy reduces to + /// DELETE+INSERT (no `MERGE INTO`) — see the `Merge` arm for why. + pub fn statements(&self) -> Vec { + let t = self.target_qualified; + let sel = self.select_sql; + let pcol = self.partition_col; + let pval = self.partition_value_sql; + let mut out = Vec::new(); + + // Whole-table replace: rebuild the table to match the SELECT's *current* + // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake + // still snapshots). This is the only path that survives a changed SELECT + // or a pre-existing table with a different schema — the persist-and- + // mutate paths below fix the schema at first create. + if !self.partitioned && matches!(self.strategy, MaterializeStrategy::Replace) { + out.push(format!( + "CREATE OR REPLACE TABLE {t} AS SELECT * FROM ({sel});" + )); + return out; + } + + // Persist-and-mutate (partitioned, or merge/append): bootstrap the table + // if absent, then write into it. The schema is fixed at first create — + // a later SELECT-schema change needs a manual rebuild (schema evolution + // is a follow-up). + if self.partitioned { + out.push(format!( + "CREATE TABLE IF NOT EXISTS {t} AS \ + SELECT *, CAST(NULL AS VARCHAR) AS {pcol} FROM ({sel}) WHERE false;" + )); + out.push(format!("ALTER TABLE {t} SET PARTITIONED BY ({pcol});")); + } else { + out.push(format!( + "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;" + )); + } + + out.push("BEGIN TRANSACTION;".to_string()); + // The rows to write, with the partition column appended when partitioned. + let source = if self.partitioned { + format!("SELECT *, {pval} AS {pcol} FROM ({sel})") + } else { + format!("SELECT * FROM ({sel})") + }; + match &self.strategy { + MaterializeStrategy::Replace => { + // Only reached when partitioned (whole-table replace returned above). + out.push(format!("DELETE FROM {t} WHERE {pcol} = {pval};")); + out.push(format!("INSERT INTO {t} {source};")); + } + MaterializeStrategy::Append => { + out.push(format!("INSERT INTO {t} {source};")); + } + MaterializeStrategy::Merge { unique_key } => { + // Upsert within the slice via delete-by-key + insert (dbt's + // `delete+insert`): rows whose key is in the incoming SELECT are + // replaced, others are left in place. This deliberately avoids + // `MERGE INTO` — DuckLake's MERGE fails writing the first rows of + // a fresh partition (HTTP 404 on the new parquet), and a failed + // write leaves the table needing a DROP. DELETE+INSERT is the + // same write shape as `replace`, which is reliable. The DELETE is + // scoped to the current partition when partitioned so it stays + // slice-local (a key present in another partition is untouched). + let scope = if self.partitioned { + format!("{pcol} = {pval} AND ") + } else { + String::new() + }; + out.push(format!( + "DELETE FROM {t} WHERE {scope}{unique_key} IN (SELECT {unique_key} FROM ({sel}));" + )); + out.push(format!("INSERT INTO {t} {source};")); + } + } + out.push("COMMIT;".to_string()); + out + } +} + +/// The read that captures the DuckLake snapshot id produced by the write, for +/// the given attach alias (e.g. `_wm_target`). The worker runs this last and +/// records the result into `materialized_partition`. +pub fn snapshot_capture_sql(alias: &str) -> String { + format!("SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('{alias}');") +} + +/// Reserved attach alias for the materialization target, fully-qualified in all +/// generated SQL so a user `USE …;` in the setup blocks can't redirect the +/// write. The worker resolves the real `ATTACH 'ducklake:…' AS _wm_target (…)` +/// from the target ducklake's config and passes it in as `target_attach`. +pub const TARGET_ALIAS: &str = "_wm_target"; + +/// Assemble the full ordered statement list the DuckDB executor runs for a +/// managed `// materialize` script. This is the single entry point the worker +/// calls; it composes the already-tested pieces (classifier split → target +/// ATTACH → strategy codegen → snapshot capture) so their ordering lives in one +/// tested place rather than inline in the executor. +/// +/// `target_attach` is the real `ATTACH 'ducklake:…' AS _wm_target (…);` string +/// the worker built from config (it depends on resolved credentials, so it +/// can't be generated here). `target_table` is the table within that catalog +/// (e.g. `orders_daily`), referenced as `_wm_target.
`. `asset_path` is +/// the full `/
` for the result summary. The trailing statement is +/// a one-row summary read (asset / rows / snapshot_id) that is both the job's +/// result (a useful preview) and what the worker records. +pub fn build_wrap_blocks( + plan: &WrapPlan, + target_attach: &str, + target_table: &str, + asset_path: &str, + partition_col: &str, + partition_value_sql: &str, + partitioned: bool, + strategy: MaterializeStrategy, +) -> Vec { + let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); + let cg = MaterializeCodegen { + target_qualified: &target_qualified, + select_sql: &plan.output, + partition_col, + partition_value_sql, + partitioned, + strategy, + }; + let mut blocks: Vec = Vec::new(); + // Setup blocks come from the splitter with their `;` stripped — re-terminate + // each so that when the executor re-joins and re-splits the assembled query, + // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH) + // don't merge into one malformed statement. + blocks.extend(plan.setup.iter().map(|s| terminate(s))); + blocks.push(target_attach.to_string()); + blocks.extend(cg.statements()); + blocks.push(materialize_result_sql( + &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + )); + blocks +} + +/// The trailing one-row summary the materialize run returns: the asset it +/// produced, the row count of the materialized slice (the partition when +/// partitioned, else the whole table), and the DuckLake snapshot it created. +/// This is both a useful preview result and the row the worker records. +pub fn materialize_result_sql( + target_qualified: &str, + asset_path: &str, + partition_col: &str, + partition_value_sql: &str, + partitioned: bool, +) -> String { + let (count_expr, partition_sel) = if partitioned { + // Row count is the slice this run wrote (the partition); `partition` + // lets the UI label the count and scope the preview to it. + ( + format!( + "(SELECT count(*) FROM {target_qualified} WHERE {partition_col} = {partition_value_sql})" + ), + format!("{partition_value_sql} AS partition, "), + ) + } else { + ( + format!("(SELECT count(*) FROM {target_qualified})"), + String::new(), + ) + }; + format!( + "SELECT 'ducklake://{asset_path}' AS materialized, \ + {partition_sel}{count_expr} AS rows, \ + (SELECT max(snapshot_id) FROM ducklake_snapshots('{TARGET_ALIAS}')) AS snapshot_id;" + ) +} + +// Ensure a statement ends with a single `;`. +fn terminate(stmt: &str) -> String { + let t = stmt.trim_end(); + if t.ends_with(';') { + t.to_string() + } else { + format!("{t};") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ok(sql: &str) -> WrapPlan { + classify_wrap(sql).expect("expected wrap-eligible") + } + fn err(sql: &str) -> WrapError { + classify_wrap(sql).expect_err("expected wrap-ineligible") + } + + #[test] + fn split_respects_strings_comments_idents() { + let sql = "SET x=1; -- a; comment\nSELECT ';' AS a, \"weird;col\" /* ; */ FROM t;"; + let s = split_statements(sql); + assert_eq!(s.len(), 2); + assert_eq!(s[0], "SET x=1"); + assert!(s[1].starts_with("SELECT")); + assert!(s[1].contains("\"weird;col\"")); + } + + #[test] + fn split_handles_escaped_quote() { + let s = split_statements("SELECT 'it''s; fine' AS a;"); + assert_eq!(s.len(), 1); + assert!(s[0].contains("it''s; fine")); + } + + #[test] + fn pipeline_annotations_are_stripped() { + // The real shape: `//` annotation lines above the SQL must not pollute + // the first block's classification (regression — they were being read + // as a leading `pipeline` keyword and rejected). + let p = ok("// pipeline\n// materialize ducklake://main/t\n// partitioned daily\nATTACH 'ducklake://main' AS dl;\nSELECT 1 AS id"); + assert_eq!(p.setup.len(), 1); + // The annotation lines are gone — the setup block starts at the real + // SQL (the `//` inside `ducklake://main` is legitimately retained). + assert!(p.setup[0].starts_with("ATTACH")); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn bare_select_is_eligible() { + let p = ok("SELECT a, b FROM t WHERE c = '{partition}'"); + assert!(p.setup.is_empty()); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn setup_then_select_is_eligible() { + let p = ok( + "ATTACH 'ducklake://main' AS dl;\n SET memory_limit='4GB';\n SELECT * FROM dl.orders", + ); + assert_eq!(p.setup.len(), 2); + assert!(p.output.starts_with("SELECT")); + } + + #[test] + fn create_temp_staging_is_setup() { + let p = ok("CREATE TEMP TABLE s AS SELECT 1; SELECT * FROM s"); + assert_eq!(p.setup.len(), 1); + assert_eq!( + classify_block("CREATE TEMP TABLE s AS SELECT 1"), + BlockClass::Setup + ); + assert_eq!( + classify_block("CREATE OR REPLACE TEMPORARY VIEW v AS SELECT 1"), + BlockClass::Setup + ); + } + + #[test] + fn with_cte_select_is_output_write_is_disallowed() { + assert_eq!( + classify_block("WITH x AS (SELECT 1) SELECT * FROM x"), + BlockClass::Output + ); + // CTE whose main statement inserts is a write, even though it starts WITH. + assert_eq!( + classify_block("WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x"), + BlockClass::Disallowed + ); + } + + #[test] + fn from_first_and_values_are_output() { + assert_eq!(classify_block("FROM t SELECT a"), BlockClass::Output); + assert_eq!(classify_block("VALUES (1),(2)"), BlockClass::Output); + assert_eq!(classify_block("TABLE t"), BlockClass::Output); + } + + #[test] + fn trailing_write_rejected() { + assert_eq!( + err("SELECT * FROM t; INSERT INTO u VALUES (1)"), + WrapError::OutputNotLast + ); + } + + #[test] + fn write_in_preamble_rejected() { + match err("INSERT INTO t VALUES (1); SELECT * FROM t") { + WrapError::DisallowedBlock { snippet } => assert!(snippet.starts_with("INSERT")), + e => panic!("wrong error: {e:?}"), + } + } + + #[test] + fn multiple_selects_rejected() { + assert_eq!( + err("SELECT 1; SELECT 2"), + WrapError::MultipleOutputs { count: 2 } + ); + } + + #[test] + fn no_select_and_empty_rejected() { + assert_eq!(err("CREATE TABLE t (a INT)"), WrapError::NoOutput); + assert_eq!(err(" -- just a comment\n"), WrapError::Empty); + } + + #[test] + fn use_cannot_redirect_is_classified_setup() { + // `USE` is allowed setup; generated SQL is fully qualified regardless. + assert_eq!(classify_block("USE dl"), BlockClass::Setup); + } + + #[test] + fn codegen_replace_partitioned() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT a FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Replace, + }; + let st = cg.statements(); + assert!(st[0].contains("CREATE TABLE IF NOT EXISTS _wm_target.orders_daily")); + assert!(st[0].contains("CAST(NULL AS VARCHAR) AS _wm_partition")); + assert!(st.iter().any( + |s| s == "ALTER TABLE _wm_target.orders_daily SET PARTITIONED BY (_wm_partition);" + )); + assert!(st.iter().any(|s| s.starts_with( + "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'" + ))); + assert!(st.iter().any(|s| s.contains( + "INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19' AS _wm_partition" + ))); + assert_eq!(st.first().map(|_| &st[st.len() - 1]).unwrap(), "COMMIT;"); + } + + #[test] + fn codegen_merge_is_delete_by_key_plus_insert() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT order_id, amount FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + }; + let st = cg.statements(); + // upsert = delete-by-key (partition-scoped) + insert — NO `MERGE INTO` + // (DuckLake's MERGE fails on fresh partitions). + assert!(!st.iter().any(|s| s.contains("MERGE INTO"))); + let del = st + .iter() + .find(|s| s.starts_with("DELETE FROM")) + .expect("delete stmt"); + assert!(del.contains( + "WHERE _wm_partition = '2026-06-19' AND order_id IN (SELECT order_id FROM (SELECT order_id, amount FROM dl.orders))" + )); + assert!(st + .iter() + .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'"))); + } + + #[test] + fn codegen_append_inserts_only() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.events", + select_sql: "SELECT * FROM dl.raw", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: true, + strategy: MaterializeStrategy::Append, + }; + let st = cg.statements(); + assert!(st + .iter() + .any(|s| s.starts_with("INSERT INTO _wm_target.events"))); + assert!(!st.iter().any(|s| s.starts_with("DELETE"))); + assert!(!st.iter().any(|s| s.starts_with("MERGE"))); + } + + #[test] + fn codegen_whole_table_replace_is_create_or_replace() { + // Unpartitioned replace must use CREATE OR REPLACE so a changed SELECT + // schema (or a pre-existing table with a different schema) doesn't break + // — and nothing else (no bootstrap / DELETE / INSERT / txn). + let cg = MaterializeCodegen { + target_qualified: "_wm_target.customer_dim", + select_sql: "SELECT a, b, c FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Replace, + }; + let st = cg.statements(); + assert_eq!( + st, + vec![ + "CREATE OR REPLACE TABLE _wm_target.customer_dim AS SELECT * FROM (SELECT a, b, c FROM dl.src);" + .to_string() + ] + ); + } + + #[test] + fn snapshot_capture_targets_alias() { + assert_eq!( + snapshot_capture_sql("_wm_target"), + "SELECT max(snapshot_id) AS snapshot_id FROM ducklake_snapshots('_wm_target');" + ); + } + + #[test] + fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() { + let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'"); + let blocks = build_wrap_blocks( + &plan, + "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');", + "orders_daily", + "main/orders_daily", + "_wm_partition", + "'2026-06-19'", + true, + MaterializeStrategy::Replace, + ); + // setup block first, then the target ATTACH, then codegen, then result. + assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl")); + // every setup block must be `;`-terminated so re-splitting can't merge it + // with the synthetic target ATTACH that follows. + assert!(blocks[0].ends_with(';')); + assert_eq!( + blocks[1], + "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');" + ); + assert!(blocks.iter().any(|b| b.contains("_wm_target.orders_daily"))); + assert!(blocks.iter().any(|b| b.starts_with( + "DELETE FROM _wm_target.orders_daily WHERE _wm_partition = '2026-06-19'" + ))); + // the trailing block is the one-row summary (asset / rows / snapshot_id), + // partition-scoped for the row count + let last = blocks.last().unwrap(); + assert!(last.contains("'ducklake://main/orders_daily' AS materialized")); + assert!(last.contains("'2026-06-19' AS partition")); + assert!(last.contains("WHERE _wm_partition = '2026-06-19') AS rows")); + assert!(last.contains("ducklake_snapshots('_wm_target')")); + } +} diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json new file mode 100644 index 0000000000..083ee7a8ba --- /dev/null +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -0,0 +1,305 @@ +[ + { + "name": "bare pipeline marker", + "code": "// pipeline\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "pipeline keyword with trailing text is not a marker", + "code": "// pipeline for billing\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "asset triggers across kinds, duplicates deduped", + "code": "-- pipeline\n-- on datatable://main/orders\n-- on s3://bucket/raw.parquet\n-- on ducklake://lake/events\n-- on datatable://main/orders\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "datatable:main/orders", + "s3object:bucket/raw.parquet", + "ducklake:lake/events" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "native trigger markers", + "code": "# pipeline\n# on kafka\n# on schedule\n# on data_upload\nprint(1)", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": ["kafka", "schedule", "data_upload"], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "native keyword must be a whole word and end the line", + "code": "// on kafkalike\n// on kafka topic-extra\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned daily with options", + "code": "// pipeline\n// partitioned daily tz=Europe/Paris format=%Y-%m-%d start=2024-01-01\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": { + "kind": "daily", + "tz": "Europe/Paris", + "format": "%Y-%m-%d", + "start": "2024-01-01" + }, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned dynamic requires key, quoted values parse", + "code": "// partitioned dynamic key=\"customer id\"\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": { + "kind": "dynamic", + "key": "customer id", + "tz": null, + "format": null, + "start": null + }, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned with unknown kind is rejected", + "code": "// partitioned fortnightly\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned keyword does not match partition prefix", + "code": "// partition daily\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "freshness and tag, first value wins on duplicates", + "code": "# pipeline\n# freshness 1h\n# freshness 2h\n# tag heavy\n# tag light\nprint(1)", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": "1h", + "tag": "heavy", + "retry": null + } + }, + { + "name": "retry with count and delay", + "code": "// pipeline\n// retry 3 5s\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": { "count": 3, "delay": "5s" } + } + }, + { + "name": "retry count only", + "code": "// retry 2\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": { "count": 2, "delay": null } + } + }, + { + "name": "retry zero or malformed count is rejected", + "code": "// retry 0\n// retry 3foo\n// retry -1\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "annotations only count inside comments", + "code": "on datatable://main/raw\npipeline\nconst s = 'on s3://bucket/x'\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partition token preserved in trigger path", + "code": "// pipeline\n// on s3://bucket/daily/{partition}/data.parquet\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": ["s3object:bucket/daily/{partition}/data.parquet"], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "leading whitespace and mixed comment prefixes", + "code": " -- pipeline\n\t-- on datatable://main/x\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": ["datatable:main/x"], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "on with empty or unparseable spec is ignored", + "code": "// on\n// on \n// on notaprefix/foo\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "materialize managed (default) with merge key", + "code": "// pipeline\n// materialize ducklake://analytics/orders_daily key=order_id\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders_daily", + "unique_key": "order_id" + } + } + }, + { + "name": "materialize manual escape hatch, first value wins", + "code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders_daily", + "manual": true + } + } + }, + { + "name": "materialize default-syntax shorthand with append", + "code": "// materialize ducklake append\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "main", + "append": true + } + } + }, + { + "name": "materialize manual with no target is dropped", + "code": "// materialize manual\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + } +] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs new file mode 100644 index 0000000000..5e863fc06e --- /dev/null +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -0,0 +1,193 @@ +//! Parser-parity guard: `parse_pipeline_annotations` (Rust, drives deploy) +//! and `parsePipelineAnnotations` (TS, drives the live graph preview — +//! frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts) +//! must stay behaviorally identical, or the graph the user previews is not +//! the graph that deploys. Both implementations run the SAME fixture corpus: +//! +//! tests/fixtures/pipeline_annotations.json +//! +//! The frontend counterpart is parsePipelineAnnotations.parity.test.ts. +//! When the annotation grammar changes, extend the corpus — a fixture that +//! passes on one side and fails on the other is exactly the drift this +//! exists to catch. Only the fields both parsers produce are compared +//! (join_mode / debounce_default are deploy-only, parsed solely in Rust). + +use serde::Deserialize; +use windmill_parser::asset_parser::{ + parse_pipeline_annotations, AssetKind, PartitionKind, TriggerSpec, +}; + +#[derive(Deserialize)] +struct Fixture { + name: String, + code: String, + expected: Expected, +} + +#[derive(Deserialize)] +struct Expected { + in_pipeline: bool, + /// `kind:path`, in declaration order, deduped. + asset_triggers: Vec, + native_triggers: Vec, + partition: Option, + freshness: Option, + tag: Option, + retry: Option, + // Default-on-absent so the pre-existing fixtures (which omit it) keep + // deserializing; only fixtures exercising materialization set it. + #[serde(default)] + materialize: Option, +} + +#[derive(Deserialize)] +struct ExpectedMaterialize { + target_kind: String, + target_path: String, + #[serde(default)] + manual: bool, + #[serde(default)] + append: bool, + #[serde(default)] + unique_key: Option, +} + +#[derive(Deserialize)] +struct ExpectedPartition { + kind: String, + #[serde(default)] + key: Option, + tz: Option, + format: Option, + start: Option, +} + +#[derive(Deserialize)] +struct ExpectedRetry { + count: u32, + delay: Option, +} + +fn kind_str(k: AssetKind) -> &'static str { + match k { + AssetKind::S3Object => "s3object", + AssetKind::Resource => "resource", + AssetKind::Ducklake => "ducklake", + AssetKind::DataTable => "datatable", + AssetKind::Volume => "volume", + } +} + +fn native_str(t: &TriggerSpec) -> Option<&'static str> { + Some(match t { + TriggerSpec::Asset { .. } => return None, + TriggerSpec::Schedule => "schedule", + TriggerSpec::Webhook => "webhook", + TriggerSpec::Email => "email", + TriggerSpec::Kafka => "kafka", + TriggerSpec::Mqtt => "mqtt", + TriggerSpec::Nats => "nats", + TriggerSpec::Postgres => "postgres", + TriggerSpec::Sqs => "sqs", + TriggerSpec::Gcp => "gcp", + TriggerSpec::DataUpload => "data_upload", + }) +} + +#[test] +fn pipeline_annotation_fixtures_match() { + let fixtures: Vec = + serde_json::from_str(include_str!("fixtures/pipeline_annotations.json")) + .expect("fixture corpus must deserialize"); + assert!(!fixtures.is_empty()); + + for f in fixtures { + let got = parse_pipeline_annotations(&f.code); + let ctx = format!("fixture '{}'", f.name); + + assert_eq!( + got.in_pipeline, f.expected.in_pipeline, + "{ctx}: in_pipeline" + ); + + let asset_triggers: Vec = got + .triggers + .iter() + .filter_map(|t| match t { + TriggerSpec::Asset { asset_kind, path, .. } => { + Some(format!("{}:{}", kind_str(*asset_kind), path)) + } + _ => None, + }) + .collect(); + assert_eq!( + asset_triggers, f.expected.asset_triggers, + "{ctx}: asset triggers" + ); + + let native: Vec<&str> = got.triggers.iter().filter_map(native_str).collect(); + assert_eq!(native, f.expected.native_triggers, "{ctx}: native triggers"); + + match (&got.partition, &f.expected.partition) { + (None, None) => {} + (Some(p), Some(e)) => { + let (kind, key) = match &p.kind { + PartitionKind::Daily => ("daily", None), + PartitionKind::Hourly => ("hourly", None), + PartitionKind::Weekly => ("weekly", None), + PartitionKind::Monthly => ("monthly", None), + PartitionKind::Dynamic { key } => ("dynamic", Some(key.clone())), + }; + assert_eq!(kind, e.kind, "{ctx}: partition kind"); + assert_eq!(key, e.key, "{ctx}: partition key"); + assert_eq!(p.tz, e.tz, "{ctx}: partition tz"); + assert_eq!(p.format, e.format, "{ctx}: partition format"); + assert_eq!(p.start, e.start, "{ctx}: partition start"); + } + (got, want) => panic!( + "{ctx}: partition mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } + + assert_eq!( + got.freshness.as_ref().map(|fr| fr.duration.clone()), + f.expected.freshness, + "{ctx}: freshness" + ); + assert_eq!(got.tag, f.expected.tag, "{ctx}: tag"); + match (&got.retry, &f.expected.retry) { + (None, None) => {} + (Some(r), Some(e)) => { + assert_eq!(r.count, e.count, "{ctx}: retry count"); + assert_eq!(r.delay, e.delay, "{ctx}: retry delay"); + } + (got, want) => panic!( + "{ctx}: retry mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } + + match (&got.materialize, &f.expected.materialize) { + (None, None) => {} + (Some(m), Some(e)) => { + assert_eq!( + kind_str(m.target_kind), + e.target_kind, + "{ctx}: materialize kind" + ); + assert_eq!(m.target_path, e.target_path, "{ctx}: materialize path"); + assert_eq!(m.manual, e.manual, "{ctx}: materialize manual"); + assert_eq!(m.append, e.append, "{ctx}: materialize append"); + assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key"); + } + (got, want) => panic!( + "{ctx}: materialize mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index a97320d5d9..57cc02598a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1646,6 +1646,13 @@ async fn process_notify_event( ); windmill_common::variables::CUSTOM_ENVS_CACHE.remove(payload); } + "notify_asset_producer_change" => { + tracing::debug!( + "Asset producer change for workspace {}, invalidating producer-writes cache", + payload + ); + windmill_queue::asset_dispatch::ASSET_PRODUCER_WRITES_CACHE.remove(payload); + } "notify_workspace_key_change" => { tracing::info!( "Workspace key change detected, invalidating workspace key cache: {}", diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f396726087..be54edf468 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1275,6 +1275,10 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting autoscaling event on CE: {:?}", e); } + if let Err(e) = windmill_queue::cascade::reap_stale_join_slots(db).await { + tracing::error!("Error reaping stale join_pending_inputs slots: {:?}", e); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) diff --git a/backend/tests/asset_trigger_dispatch.rs b/backend/tests/asset_trigger_dispatch.rs new file mode 100644 index 0000000000..e79937f702 --- /dev/null +++ b/backend/tests/asset_trigger_dispatch.rs @@ -0,0 +1,999 @@ +//! End-to-end test for asset-trigger dispatch. +//! +//! Runs a real Bash producer through a worker, lets the +//! `result_processor` hook fire `dispatch_asset_triggers`, and then makes +//! several follow-up calls into `dispatch_asset_triggers` against the same +//! seeded graph to cover the eligibility branches (self-loop, skip arg, +//! cycle guard, flow subscriber, ineligible job kinds). Direct calls share +//! the same workspace so we exercise the real query paths against real +//! `asset` / `script_trigger` rows produced by deploy-equivalent seeding. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::jobs::{JobKind, JobPayload}; +use windmill_common::scripts::{ScriptHash, ScriptLang}; +use windmill_queue::asset_dispatch::dispatch_asset_triggers; +use windmill_queue::cascade::reap_stale_join_slots; +use windmill_queue::MiniCompletedJob; +use windmill_test_utils::{initialize_tracing, ApiServer, RunJob}; + +const WS: &str = "test-workspace"; +const PRODUCER: &str = "u/test-user/producer"; +const SUB_S3: &str = "u/test-user/sub-s3"; +const SUB_RES: &str = "u/test-user/sub-res"; +const SUB_FLOW: &str = "u/test-user/sub-flow"; + +// ── Seeding helpers ─────────────────────────────────────────────────────── + +async fn seed_script( + db: &Pool, + path: &str, + content: &str, + language: &str, +) -> anyhow::Result { + // Hash needs to be unique per (workspace, hash). Derive from path AND + // content: the worker's script cache (`cache::script::fetch`) is keyed + // by hash alone and is process-global, so tests running in the same + // process that seed the same path with different content would poison + // each other's cache if the hash came from the path only. + let mut h = 0i64; + for b in path.bytes().chain(content.bytes()) { + h = h.wrapping_mul(31).wrapping_add(b as i64); + } + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, tag, lock) + VALUES ($1, $2, $3, '', '', $4, 'test-user', $5::script_lang, 'deno', '') + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(h) + .bind(path) + .bind(content) + .bind(language) + .execute(db) + .await?; + Ok(h) +} + +async fn seed_asset_write( + db: &Pool, + producer_path: &str, + kind: &str, + asset_path: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, $3::asset_kind, 'w'::asset_access_type, $4, 'script'::asset_usage_kind) + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(asset_path) + .bind(kind) + .bind(producer_path) + .execute(db) + .await?; + // These tests use #[sqlx::test] isolated DBs that all share one workspace + // id, so the process-global producer cache (keyed by workspace) would + // clobber across DBs under concurrent test threads. Disable it so every + // dispatch reads the test's own DB. (Production invalidates via the + // notify_event poller instead.) + windmill_queue::asset_dispatch::ASSET_PRODUCER_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) +} + +async fn seed_subscription( + db: &Pool, + subscriber_path: &str, + subscriber_kind: &str, + trigger_ref: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref) + VALUES ($1, $2::asset_usage_kind, $3, 'asset'::script_trigger_kind, $4)"#, + ) + .bind(WS) + .bind(subscriber_kind) + .bind(subscriber_path) + .bind(trigger_ref) + .execute(db) + .await?; + Ok(()) +} + +/// Insert a synthetic producer `v2_job` row used by the direct +/// `dispatch_asset_triggers` calls in the edge-case section. `args` lets the +/// test inject `_wmill_skip_asset_dispatch` or `trigger.chain` so the +/// dispatcher's arg-driven branches are exercised against real rows. +async fn seed_producer_job(db: &Pool, args: serde_json::Value) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query!( + r#"INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by, + permissioned_as, permissioned_as_email, tag, script_lang) + VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user', + 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)"#, + id, + WS, + PRODUCER, + args, + ) + .execute(db) + .await?; + Ok(id) +} + +/// Like `seed_producer_job` but for an arbitrary runnable path (the +/// AND-join test needs two distinct producers). +async fn seed_producer_job_path( + db: &Pool, + path: &str, + args: serde_json::Value, +) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query!( + r#"INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by, + permissioned_as, permissioned_as_email, tag, script_lang) + VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user', + 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)"#, + id, + WS, + path, + args, + ) + .execute(db) + .await?; + Ok(id) +} + +/// Seed an asset subscription flagged as an AND join (`// trigger all`). +async fn seed_subscription_and( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, TRUE)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .execute(db) + .await?; + Ok(()) +} + +/// Seed an asset subscription with an opt-in debounce window (seconds). +async fn seed_subscription_debounced( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, + debounce_s: i32, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, debounce_s) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, $4)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .bind(debounce_s) + .execute(db) + .await?; + Ok(()) +} + +/// Seed an asset subscription with a `// retry []` policy. +async fn seed_subscription_with_retry( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, + retry_count: i16, + retry_delay_s: i32, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, + retry_count, retry_delay_s) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, $4, $5)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .bind(retry_count) + .bind(retry_delay_s) + .execute(db) + .await?; + Ok(()) +} + +fn make_mini(id: Uuid, runnable_path: &str) -> MiniCompletedJob { + MiniCompletedJob { + id, + workspace_id: WS.to_string(), + runnable_id: Some(ScriptHash(1)), + scheduled_for: chrono::Utc::now(), + parent_job: None, + flow_innermost_root_job: None, + runnable_path: Some(runnable_path.to_string()), + kind: JobKind::Script, + started_at: Some(chrono::Utc::now()), + permissioned_as: "u/test-user".to_string(), + created_by: "test-user".to_string(), + script_lang: Some(ScriptLang::Bash), + permissioned_as_email: "test@windmill.dev".to_string(), + flow_step_id: None, + trigger_kind: None, + trigger: None, + priority: None, + concurrent_limit: None, + tag: "deno".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + runnable_settings_handle: None, + } +} + +/// Read `v2_job` rows that were created by asset dispatch (filtered by +/// `trigger_kind = 'asset'` so dep-jobs / other infra rows don't leak in). +async fn fetch_dispatched( + db: &Pool, +) -> anyhow::Result, Option)>> { + let rows = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", trigger, + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'asset' + ORDER BY runnable_path"#, + WS, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.runnable_path, r.trigger, r.args.map(|j| j.0))) + .collect()) +} + +async fn clear_dispatched(db: &Pool) -> anyhow::Result<()> { + sqlx::query!( + "DELETE FROM v2_job WHERE workspace_id = $1 AND trigger_kind = 'asset'", + WS, + ) + .execute(db) + .await?; + Ok(()) +} + +// ── The test ───────────────────────────────────────────────────────────── + +/// One end-to-end test that: +/// 1. seeds a producer that writes two asset kinds (s3 + resource) with three +/// subscribers (two script subs + one flow sub that must be skipped), +/// 2. runs the producer through a real worker and asserts the +/// `result_processor` hook fired and pushed the right jobs with the +/// right trigger metadata, +/// 3. then drives `dispatch_asset_triggers` directly against the same +/// seeded graph to cover the arg-driven and eligibility branches that +/// can't be reached by varying the producer's runtime args alone: +/// - skip arg suppresses dispatch +/// - a subscriber already in the lineage is skipped (cycle guard) +/// - the lineage chain accumulates the producer path each hop +/// - self-loop subscriber is filtered +/// - producer with parent_job is ineligible +/// - producer with `Flow` kind is ineligible +#[sqlx::test(fixtures("base"))] +async fn end_to_end_asset_dispatch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // ── Seed the graph ────────────────────────────────────────────────── + let producer_hash = seed_script(&db, PRODUCER, "echo producer", "bash").await?; + seed_script(&db, SUB_S3, "echo s3-subscriber", "bash").await?; + seed_script(&db, SUB_RES, "echo res-subscriber", "bash").await?; + // Self-loop subscriber: same path as producer → must be filtered. + seed_subscription(&db, PRODUCER, "script", "s3://f/blob").await?; + // Flow subscriber on the same asset → V1 hard-filters runnable_kind='flow'. + seed_subscription(&db, SUB_FLOW, "flow", "s3://f/blob").await?; + // Legit subscribers. + seed_subscription(&db, SUB_S3, "script", "s3://f/blob").await?; + seed_subscription(&db, SUB_RES, "script", "$res:f/cfg").await?; + // Two writes from one producer, distinct kinds. + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_asset_write(&db, PRODUCER, "resource", "f/cfg").await?; + + // ── 1. Real worker run: the hook must fire after producer success ─── + let job = JobPayload::ScriptHash { + path: PRODUCER.to_string(), + hash: ScriptHash(producer_hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bash, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }; + let completed = RunJob::from(job).run_until_complete(&db, false, port).await; + assert!( + completed.success, + "producer must succeed for dispatch to fire" + ); + + let mut rows = fetch_dispatched(&db).await?; + rows.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + rows.len(), + 2, + "expected dispatch to the two legit script subscribers (flow sub filtered, self-loop filtered)" + ); + let by_path: std::collections::HashMap<_, _> = rows.iter().map(|r| (r.0.as_str(), r)).collect(); + + let s3_row = by_path + .get(SUB_S3) + .expect("s3 subscriber should have a job"); + let s3_trig = s3_row.2.as_ref().unwrap().get("trigger").unwrap(); + assert_eq!(s3_trig["kind"], "asset"); + assert_eq!(s3_trig["asset_kind"], "s3object"); + assert_eq!(s3_trig["asset_path"], "f/blob"); + assert_eq!(s3_trig["producer_path"], PRODUCER); + assert_eq!( + s3_trig["chain"], + json!([PRODUCER]), + "lineage starts with the producer on the first hop" + ); + assert_eq!(s3_row.1.as_deref(), Some(PRODUCER)); + + let res_row = by_path + .get(SUB_RES) + .expect("resource subscriber should have a job"); + let res_trig = res_row.2.as_ref().unwrap().get("trigger").unwrap(); + assert_eq!(res_trig["asset_kind"], "resource"); + assert_eq!(res_trig["asset_path"], "f/cfg"); + + // ── 2. Direct calls to dispatch_asset_triggers for arg / eligibility + // branches that can't be reached via the runtime path ────────── + clear_dispatched(&db).await?; + + // skip arg suppresses dispatch + let id = seed_producer_job(&db, json!({ "_wmill_skip_asset_dispatch": true })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 0, "skip arg suppressed dispatch"); + + // cycle guard: a subscriber already in the lineage is skipped, but its + // siblings still dispatch (only the cyclic edge is cut). + let id = seed_producer_job(&db, json!({ "trigger": { "chain": [SUB_S3] } })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!( + r.dispatched.len(), + 1, + "cyclic subscriber (already in lineage) skipped; sibling still dispatched" + ); + + // lineage accumulates: a fresh producer extends the chain with its own path + clear_dispatched(&db).await?; + let id = seed_producer_job(&db, json!({ "trigger": { "chain": ["f/upstream"] } })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2); + let rows = fetch_dispatched(&db).await?; + for row in &rows { + assert_eq!( + row.2.as_ref().unwrap()["trigger"]["chain"], + json!(["f/upstream", PRODUCER]), + "lineage accumulates the producer path" + ); + } + + // producer with parent_job (flow step) is ineligible + clear_dispatched(&db).await?; + let id = seed_producer_job(&db, json!({})).await?; + let mut mini = make_mini(id, PRODUCER); + mini.parent_job = Some(Uuid::new_v4()); + let r = dispatch_asset_triggers(&db, &mini).await; + assert_eq!(r.dispatched.len(), 0, "flow step producer ineligible"); + + // producer with kind=Flow is ineligible + let id = seed_producer_job(&db, json!({})).await?; + let mut mini = make_mini(id, PRODUCER); + mini.kind = JobKind::Flow; + let r = dispatch_asset_triggers(&db, &mini).await; + assert_eq!(r.dispatched.len(), 0, "flow producer ineligible"); + + // Sanity: the eligible direct call (clean producer, no args) still fires — + // proves the assertions above are negative cases, not a broken setup. + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "clean direct call still dispatches"); + + Ok(()) +} + +/// Stage C: a `// partitioned dynamic` producer run through a real worker +/// must (1) resolve the partition off its triggering payload at execution +/// time, (2) persist it back into its own `v2_job.args` so the cascade +/// reads it, and (3) propagate the same value into the dispatched +/// subscriber's args + `trigger.partition`. +#[sqlx::test(fixtures("base"))] +async fn partition_dynamic_resolved_persisted_and_propagated( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Producer declares a dynamic partition keyed off the run payload. + // (Bash uses `#` comments — the annotation parser accepts `#`/`--`/`//`.) + let producer_hash = seed_script( + &db, + PRODUCER, + "# pipeline\n# partitioned dynamic key=\"$.tenant_id\"\necho producer", + "bash", + ) + .await?; + seed_script(&db, SUB_S3, "echo s3-subscriber", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_subscription(&db, SUB_S3, "script", "s3://f/blob").await?; + + let job = JobPayload::ScriptHash { + path: PRODUCER.to_string(), + hash: ScriptHash(producer_hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bash, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }; + let completed = RunJob::from(job) + .arg("tenant_id", json!("acme")) + .run_until_complete(&db, false, port) + .await; + assert!(completed.success, "partitioned producer must succeed"); + + // (2) resolved value persisted back into the producer's own args. + let prod = sqlx::query!( + r#"SELECT args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND runnable_path = $2 AND trigger_kind IS NULL"#, + WS, + PRODUCER, + ) + .fetch_one(&db) + .await?; + assert_eq!( + prod.args.unwrap().0["partition"], + json!("acme"), + "Stage C must persist the resolved partition into v2_job.args" + ); + + // (3) propagated into the dispatched subscriber. + let rows = fetch_dispatched(&db).await?; + let sub = rows + .iter() + .find(|r| r.0 == SUB_S3) + .expect("subscriber must be dispatched"); + let args = sub.2.as_ref().unwrap(); + assert_eq!( + args["partition"], + json!("acme"), + "subscriber gets top-level partition arg" + ); + assert_eq!( + args["trigger"]["partition"], + json!("acme"), + "subscriber gets trigger.partition" + ); + + Ok(()) +} + +/// Stage D: an AND-join subscriber (`// trigger all`) with two +/// partition-bearing inputs must NOT dispatch until both inputs have +/// arrived for the *same* partition; then it fires exactly once. Slots +/// are per-partition and cleared on fire (re-accumulate, no double-fire). +#[sqlx::test(fixtures("base"))] +async fn and_join_waits_for_all_partition_inputs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const PROD_A: &str = "u/test-user/prod-a"; + const PROD_B: &str = "u/test-user/prod-b"; + const SUB_J: &str = "u/test-user/sub-join"; + + seed_script(&db, SUB_J, "echo join-subscriber", "bash").await?; + // Two partition-bearing producers, one input each (literal token form). + seed_asset_write(&db, PROD_A, "s3object", "lake/{partition}/a").await?; + seed_asset_write(&db, PROD_B, "s3object", "lake/{partition}/b").await?; + seed_subscription_and(&db, SUB_J, "s3://lake/{partition}/a").await?; + seed_subscription_and(&db, SUB_J, "s3://lake/{partition}/b").await?; + + // Input A for partition "acme" → slot 1/2, must NOT dispatch. + let a_acme = seed_producer_job_path(&db, PROD_A, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(a_acme, PROD_A)).await; + assert!( + r.dispatched.is_empty(), + "AND join must wait: only 1 of 2 inputs present" + ); + assert!( + fetch_dispatched(&db).await?.is_empty(), + "no subscriber job pushed yet" + ); + + // A different partition for input B must open its OWN slot, not + // complete acme's. + let b_globex = seed_producer_job_path(&db, PROD_B, json!({ "partition": "globex" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(b_globex, PROD_B)).await; + assert!( + r.dispatched.is_empty(), + "different partition opens a separate slot, does not complete acme" + ); + + // Input B for "acme" → acme slot now 2/2 → dispatch exactly once. + let b_acme = seed_producer_job_path(&db, PROD_B, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(b_acme, PROD_B)).await; + assert_eq!(r.dispatched.len(), 1, "AND join fires once both inputs in"); + + let rows = fetch_dispatched(&db).await?; + assert_eq!(rows.len(), 1); + let sub = &rows[0]; + assert_eq!(sub.0, SUB_J); + let args = sub.2.as_ref().unwrap(); + assert_eq!(args["partition"], json!("acme")); + assert_eq!(args["trigger"]["partition"], json!("acme")); + + // Slot cleared on fire: re-arrival of A/acme alone is 1/2 again, no + // double-fire. + clear_dispatched(&db).await?; + let a_acme2 = seed_producer_job_path(&db, PROD_A, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(a_acme2, PROD_A)).await; + assert!( + r.dispatched.is_empty(), + "slot was cleared on fire; single input must not re-fire" + ); + + Ok(()) +} + +/// Stage E3: a subscriber whose edge has a debounce window gets real +/// DebouncingSettings (delay + a (subscriber, partition) key) on the +/// dispatched job; an undebounced subscriber on the same asset gets none +/// (fan-out, unchanged). Asserts the wiring fetch→push→payload→handle; +/// the actual window-collapse is the queue subsystem's own concern. +#[sqlx::test(fixtures("base"))] +async fn debounce_setting_applied_to_dispatched_subscriber( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + seed_script(&db, SUB_S3, "echo debounced", "bash").await?; + seed_script(&db, SUB_RES, "echo plain", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_subscription_debounced(&db, SUB_S3, "s3://f/blob", 30).await?; + seed_subscription(&db, SUB_RES, "script", "s3://f/blob").await?; + + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "both subscribers dispatched"); + + // Resolve the persisted debounce window straight from this test's own + // (isolated) DB by walking the handle chain + // v2_job_queue.runnable_settings_handle → runnable_settings.debouncing_settings + // → debouncing_settings. Reading the rows directly rather than through + // `prefetch_cached_from_handle` keeps the assertion off the process-global + // runnable-settings cache (and its tempdir-backed file I/O), which is + // shared by every test running concurrently in this binary — a needless + // cross-test coupling for what is purely a "was the handle wired through to + // the queued job" check. An undebounced subscriber has a NULL handle, so + // the inner joins yield no row → (None, None). + async fn debounce_of( + db: &Pool, + path: &str, + ) -> anyhow::Result<(Option, Option)> { + use sqlx::Row; + let row = sqlx::query( + r#"SELECT ds.debounce_delay_s, ds.debounce_key + FROM v2_job j + JOIN v2_job_queue q ON q.id = j.id + JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle + JOIN debouncing_settings ds ON ds.hash = rs.debouncing_settings + WHERE j.workspace_id = $1 AND j.runnable_path = $2 + AND j.trigger_kind = 'asset'"#, + ) + .bind(WS) + .bind(path) + .fetch_optional(db) + .await?; + Ok(match row { + Some(r) => ( + r.try_get::, _>("debounce_delay_s")?, + r.try_get::, _>("debounce_key")?, + ), + None => (None, None), + }) + } + + let (deb_delay, deb_key) = debounce_of(&db, SUB_S3).await?; + assert_eq!(deb_delay, Some(30), "debounced edge → 30s window"); + assert!( + deb_key + .as_deref() + .is_some_and(|k| k.starts_with("asset-cascade:")), + "debounce key is scoped to the (subscriber, partition) cascade slot, got {deb_key:?}" + ); + + let (plain_delay, _) = debounce_of(&db, SUB_RES).await?; + assert_eq!( + plain_delay, None, + "undebounced edge → no debounce (fan-out)" + ); + + Ok(()) +} + +/// `// retry []` opts the subscriber into the flow-runtime retry +/// path. Implementation-wise, the dispatcher wraps the script in a one-step +/// flow (`JobPayload::SingleStepFlow`) so the existing flow retry machinery +/// handles re-runs. Subscribers without retry continue to be pushed as +/// `JobKind::Script` (no wrapping). +#[sqlx::test(fixtures("base"))] +async fn retry_setting_wraps_dispatched_subscriber_as_flow( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + seed_script(&db, SUB_S3, "echo retrying", "bash").await?; + seed_script(&db, SUB_RES, "echo plain", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + // Retry policy on the s3 edge; res edge stays vanilla so we also assert + // the "no retry" path keeps the cheaper ScriptHash push. + seed_subscription_with_retry(&db, SUB_S3, "s3://f/blob", 3, 5).await?; + seed_subscription(&db, SUB_RES, "script", "s3://f/blob").await?; + + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "both subscribers dispatched"); + + let kinds: Vec<(String, String)> = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", kind::text AS "kind!" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'asset' + ORDER BY runnable_path"#, + WS, + ) + .fetch_all(&db) + .await? + .into_iter() + .map(|r| (r.runnable_path, r.kind)) + .collect(); + + let s3_kind = kinds + .iter() + .find(|(p, _)| p == SUB_S3) + .map(|(_, k)| k.as_str()) + .unwrap_or(""); + let res_kind = kinds + .iter() + .find(|(p, _)| p == SUB_RES) + .map(|(_, k)| k.as_str()) + .unwrap_or(""); + + assert_eq!( + s3_kind, "singlestepflow", + "retry-bearing subscriber wraps as SingleStepFlow so flow-runtime retry kicks in" + ); + assert_eq!( + res_kind, "script", + "no-retry subscriber stays as the cheaper ScriptHash push" + ); + + Ok(()) +} + +/// Regression for the AND-join check-then-act race: when a subscriber's +/// last partition-bearing inputs complete concurrently (different workers +/// finishing different upstream producers at once), the barrier must +/// still fire the subscriber exactly once for the partition. Fires all N +/// producers' dispatch simultaneously (a barrier releases them together) +/// and asserts a single dispatch and a cleared slot. This invariant holds +/// for the transactional, advisory-locked gate regardless of interleaving; +/// a regression to a non-atomic check-then-act fails it. +#[sqlx::test(fixtures("base"))] +async fn and_join_fires_once_under_concurrent_completion(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const SUB_J: &str = "u/test-user/sub-join-conc"; + const N: usize = 5; + + seed_script(&db, SUB_J, "echo join", "bash").await?; + let mut producers = Vec::new(); + for i in 0..N { + let prod = format!("u/test-user/prod-conc-{i}"); + seed_asset_write(&db, &prod, "s3object", &format!("lake/{{partition}}/i{i}")).await?; + seed_subscription_and(&db, SUB_J, &format!("s3://lake/{{partition}}/i{i}")).await?; + let id = seed_producer_job_path(&db, &prod, json!({ "partition": "acme" })).await?; + producers.push((id, prod)); + } + + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(N)); + let mut set = tokio::task::JoinSet::new(); + for (id, prod) in producers { + let db = db.clone(); + let barrier = barrier.clone(); + set.spawn(async move { + barrier.wait().await; + dispatch_asset_triggers(&db, &make_mini(id, &prod)) + .await + .dispatched + .len() + }); + } + let mut total = 0usize; + while let Some(r) = set.join_next().await { + total += r?; + } + + assert_eq!( + total, 1, + "AND join must dispatch the subscriber exactly once under concurrent completion" + ); + let fires = fetch_dispatched(&db) + .await? + .iter() + .filter(|r| r.0 == SUB_J) + .count(); + assert_eq!(fires, 1, "exactly one subscriber job pushed"); + + let leftover = sqlx::query_scalar!( + r#"SELECT count(*) AS "n!" + FROM join_pending_inputs + WHERE workspace_id = $1 AND subscriber_path = $2"#, + WS, + SUB_J, + ) + .fetch_one(&db) + .await?; + assert_eq!(leftover, 0, "join slot cleared after fire"); + + Ok(()) +} + +/// Fuller pipeline: a partitioned chain that fans in through an AND-join +/// and then fans out over several more hops. Asserts the resolved +/// partition propagates unchanged at every hop, the chain depth +/// increments per hop, the AND barrier fires once, and a different +/// partition opens an independent slot (no cross-partition bleed) across +/// the whole multi-hop graph. +/// +/// Shape: A,B (partitioned producers) ─┐ +/// ├─▶ J (// trigger all) ─▶ C ─▶ D +/// A,B ─────────────────────────┘ +#[sqlx::test(fixtures("base"))] +async fn fuller_partitioned_join_multihop_pipeline(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const PA: &str = "u/test-user/p-a"; + const PB: &str = "u/test-user/p-b"; + const JN: &str = "u/test-user/p-join"; + const CN: &str = "u/test-user/p-c"; + const DN: &str = "u/test-user/p-d"; + + for p in [JN, CN, DN] { + seed_script(&db, p, "echo step", "bash").await?; + } + seed_asset_write(&db, PA, "s3object", "lake/{partition}/a").await?; + seed_asset_write(&db, PB, "s3object", "lake/{partition}/b").await?; + seed_asset_write(&db, JN, "s3object", "lake/{partition}/j").await?; + seed_asset_write(&db, CN, "s3object", "lake/{partition}/c").await?; + // J is an AND join over both partition-bearing inputs. + seed_subscription_and(&db, JN, "s3://lake/{partition}/a").await?; + seed_subscription_and(&db, JN, "s3://lake/{partition}/b").await?; + seed_subscription(&db, CN, "script", "s3://lake/{partition}/j").await?; + seed_subscription(&db, DN, "script", "s3://lake/{partition}/c").await?; + + // Helper: assert exactly one dispatch to `path` carrying partition + // `part`, with a cascade lineage of `chain_len` producer paths. + async fn assert_hop( + db: &Pool, + path: &str, + part: &str, + chain_len: usize, + ) -> anyhow::Result<()> { + let rows = fetch_dispatched(db).await?; + let hits: Vec<_> = rows.iter().filter(|r| r.0 == path).collect(); + assert_eq!(hits.len(), 1, "expected exactly one dispatch to {path}"); + let args = hits[0].2.as_ref().unwrap(); + assert_eq!(args["partition"], json!(part), "{path} top-level partition"); + assert_eq!( + args["trigger"]["partition"], + json!(part), + "{path} trigger.partition" + ); + assert_eq!( + args["trigger"]["chain"].as_array().map(|c| c.len()), + Some(chain_len), + "{path} lineage length" + ); + Ok(()) + } + + // day1: A arrives → J waits (1/2 partition-bearing inputs). + let pa = seed_producer_job_path( + &db, + PA, + json!({ "partition": "day1", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pa, PA)).await; + assert!(r.dispatched.is_empty(), "J must wait: only A present"); + assert!(fetch_dispatched(&db).await?.is_empty()); + + // day1: B arrives → J fires once for day1 at depth 2. + let pb = seed_producer_job_path( + &db, + PB, + json!({ "partition": "day1", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pb, PB)).await; + assert_eq!(r.dispatched.len(), 1, "J fires once when both inputs in"); + assert_hop(&db, JN, "day1", 2).await?; + clear_dispatched(&db).await?; + + // J completes for day1 → C runs for day1 at depth 3. + let jn = seed_producer_job_path( + &db, + JN, + json!({ "partition": "day1", "trigger": { "chain": ["s0", PB] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(jn, JN)).await; + assert_eq!(r.dispatched.len(), 1); + assert_hop(&db, CN, "day1", 3).await?; + clear_dispatched(&db).await?; + + // C completes for day1 → D (leaf) runs for day1 at depth 4. + let cn = seed_producer_job_path( + &db, + CN, + json!({ "partition": "day1", "trigger": { "chain": ["s0", PB, JN] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(cn, CN)).await; + assert_eq!(r.dispatched.len(), 1); + assert_hop(&db, DN, "day1", 4).await?; + clear_dispatched(&db).await?; + + // A different partition opens an independent J slot — no bleed from + // the completed day1 run. + let pa2 = seed_producer_job_path( + &db, + PA, + json!({ "partition": "day2", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pa2, PA)).await; + assert!( + r.dispatched.is_empty(), + "day2 is a separate slot; J must not fire from day1's completion" + ); + + Ok(()) +} + +/// The TTL reaper deletes abandoned AND-join slots, but keyed on the +/// slot's MOST RECENT row: a slot still receiving input (newest row +/// fresh) is never reaped even if it also has rows older than the TTL. +/// This per-slot (not per-row) property is the correctness point — it +/// prevents corrupting a join whose inputs trickle in slowly. +#[sqlx::test(fixtures("base"))] +async fn reaper_clears_only_stale_join_slots(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Seed a join_pending_inputs row with an explicit age (days old). + async fn seed_slot_row( + db: &Pool, + sub: &str, + part: &str, + tref: &str, + age_days: i64, + ) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO join_pending_inputs + (workspace_id, subscriber_path, partition, trigger_ref, received_at) + VALUES ($1, $2, $3, $4, now() - ($5::bigint::text || ' d')::interval)"#, + ) + .bind(WS) + .bind(sub) + .bind(part) + .bind(tref) + .bind(age_days) + .execute(db) + .await?; + Ok(()) + } + async fn slot_count(db: &Pool, sub: &str) -> anyhow::Result { + Ok(sqlx::query_scalar!( + r#"SELECT count(*) AS "n!" FROM join_pending_inputs + WHERE workspace_id = $1 AND subscriber_path = $2"#, + WS, + sub, + ) + .fetch_one(db) + .await?) + } + + // Stale: every row older than the 60d TTL → reaped. + seed_slot_row( + &db, + "u/test-user/sub-stale", + "p1", + "s3://x/{partition}/a", + 61, + ) + .await?; + seed_slot_row( + &db, + "u/test-user/sub-stale", + "p1", + "s3://x/{partition}/b", + 90, + ) + .await?; + // Fresh: recent → kept. + seed_slot_row( + &db, + "u/test-user/sub-fresh", + "p1", + "s3://y/{partition}/a", + 0, + ) + .await?; + // Mixed: one ancient row + one fresh row in the SAME slot. max(received_at) + // is fresh, so the whole slot must be kept (the correctness property). + seed_slot_row( + &db, + "u/test-user/sub-mixed", + "p1", + "s3://z/{partition}/a", + 120, + ) + .await?; + seed_slot_row( + &db, + "u/test-user/sub-mixed", + "p1", + "s3://z/{partition}/b", + 0, + ) + .await?; + + reap_stale_join_slots(&db).await?; + + assert_eq!( + slot_count(&db, "u/test-user/sub-stale").await?, + 0, + "stale slot reaped" + ); + assert_eq!( + slot_count(&db, "u/test-user/sub-fresh").await?, + 1, + "fresh slot kept" + ); + assert_eq!( + slot_count(&db, "u/test-user/sub-mixed").await?, + 2, + "slot with a recent row must be kept entirely (per-slot, not per-row)" + ); + + Ok(()) +} diff --git a/backend/tests/drafts_nul.rs b/backend/tests/drafts_nul.rs new file mode 100644 index 0000000000..d7314b8fb6 --- /dev/null +++ b/backend/tests/drafts_nul.rs @@ -0,0 +1,80 @@ +//! Regression test for NUL bytes in draft values. +//! +//! `draft.value` is a `json` column (not `jsonb`), so a U+0000 escape can be +//! stored and then make any `->>`/`to_jsonb` extraction raise `22P05` — one +//! poisoned draft 500'd `GET /drafts/list` (silently hiding the home-page +//! "This workspace has N drafts" banner). The fix sanitizes the value on write +//! (`update_draft` -> `strip_json_nul`) so a NUL never reaches the column; this +//! drives the real endpoint and asserts the stored + listed value is NUL-free. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(b: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + b.header("Authorization", "Bearer DNUL_ADMIN_TOKEN") +} + +#[sqlx::test(fixtures("drafts_nul"))] +async fn test_draft_write_strips_nul(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/dnul-ws"); + + // Save a draft whose summary and content carry a real NUL. + let resp = authed(client().post(format!( + "{base}/drafts/update/script/u/dnul-admin/poison" + ))) + .json(&json!({ + "value": { + "summary": "hi\u{0}there", + "path": "u/dnul-admin/poison", + "content": "x\u{0}y" + } + })) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "save should succeed: {}", + resp.text().await.unwrap_or_default() + ); + + // The stored value must be NUL-free (sanitized on write). + let stored: Value = authed(client().get(format!( + "{base}/drafts/get_own/script/u/dnul-admin/poison" + ))) + .send() + .await? + .json() + .await?; + let value = stored.get("value").expect("draft should exist"); + assert_eq!(value["summary"], "hithere"); + assert_eq!(value["content"], "xy"); + assert!( + !serde_json::to_string(value).unwrap().contains("\\u0000"), + "stored value still contains a NUL escape: {value}" + ); + + // The list endpoint uses raw `->>`; it works (200, no 500) because the + // stored data is clean, and the summary comes back stripped. + let items: Vec = authed(client().get(format!("{base}/drafts/list"))) + .send() + .await? + .json() + .await?; + let item = items + .iter() + .find(|d| d["path"] == "u/dnul-admin/poison") + .expect("saved draft should be listed"); + assert_eq!(item["summary"], "hithere"); + + Ok(()) +} diff --git a/backend/tests/fixtures/drafts_nul.sql b/backend/tests/fixtures/drafts_nul.sql new file mode 100644 index 0000000000..8f457ffbc3 --- /dev/null +++ b/backend/tests/fixtures/drafts_nul.sql @@ -0,0 +1,24 @@ +-- Fixture for the draft NUL-byte write-sanitization regression test. +-- Just a workspace + admin user + token; the test itself POSTs a draft whose +-- value carries a U+0000 and asserts it is stored (and listed) NUL-free. + +INSERT INTO workspace (id, name, owner) VALUES + ('dnul-ws', 'DNUL WS', 'dnul-admin'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES + ('dnul-ws', 'cloud', 'dnul-key'); + +INSERT INTO workspace_settings (workspace_id) VALUES + ('dnul-ws'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('dnul-ws', 'all', 'All users', '{}'); + +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('dnul-admin@windmill.dev', 'x', 'password', true, true, 'DNUL Admin', 'dnul-admin'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('dnul-ws', 'dnul-admin@windmill.dev', 'dnul-admin', true, 'Admin'); + +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('DNUL_ADMIN_TOKEN'::bytea), 'hex'), 'DNUL_ADMIN', 'DNUL_ADMIN_TOKEN', 'dnul-admin@windmill.dev', 't', true); diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 3a1550d395..0f45d09147 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1113,6 +1113,7 @@ async def main(item: str, qty: int, email: str): RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content, + tag: None, ..RawCode::default() })) .arg("item", json!("widget")) @@ -1298,6 +1299,7 @@ async def main(n: int): RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content, + tag: None, ..RawCode::default() })) .arg("n", json!(1)) diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 068555c7df..8755ebfb57 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -40,6 +40,7 @@ def main(name: str): path: Some("f/test/my_script".to_string()), language: ScriptLang::Python3, modules: Some(modules), + tag: None, ..RawCode::default() }); @@ -92,6 +93,7 @@ def main(a: int, b: int): path: Some("f/test/my_script".to_string()), language: ScriptLang::Python3, modules: Some(modules), + tag: None, ..RawCode::default() }); @@ -143,6 +145,7 @@ export function main(name: string) { path: Some("f/test/my_script".to_string()), language: ScriptLang::Bun, modules: Some(modules), + tag: None, ..RawCode::default() }); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b220416b55..4ed12d3a4a 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3990,6 +3990,261 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// Push `flow`, run it on a real worker until its first step is running, then simulate +/// `monitor::handle_zombie_jobs` reaping that step unrecoverably (its worker crashed/OOM'd) +/// by calling `handle_job_error(..., unrecoverable = true, ...)` exactly as the monitor does. +/// Returns the flow's completed result. +#[cfg(feature = "deno_core")] +async fn run_flow_until_step_running_then_fail_unrecoverably( + db: &Pool, + port: u16, + flow: FlowValue, +) -> serde_json::Value { + use std::sync::atomic::AtomicU16; + use std::sync::Arc; + use tokio::sync::mpsc; + use windmill_common::auth::create_token_for_owner; + use windmill_common::client::AuthedClient; + use windmill_common::KillpillSender; + use windmill_queue::{get_queued_job_v2, MiniCompletedJob, SameWorkerPayload}; + use windmill_worker::{JobCompletedSender, SameWorkerSender}; + + let flow_id = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .push(db) + .await; + + let db_ = db.clone(); + in_test_worker( + db, + async move { + let db = db_; + + // Wait for the first step to be running on the worker. + let step_job = loop { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let running = sqlx::query_scalar!( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) + WHERE j.parent_job = $1 AND q.running = true", + flow_id + ) + .fetch_optional(&db) + .await + .unwrap(); + if let Some(step_id) = running { + if let Some(job) = get_queued_job_v2(&db, &step_id).await.unwrap() { + break job; + } + } + }; + + // The dummy `same_worker_tx` mirrors the monitor, which has no live worker channel. + let (sw_tx, _sw_rx) = mpsc::channel::(1); + let sw_tx = SameWorkerSender(sw_tx, Arc::new(AtomicU16::new(0))); + let (jc_tx, _jc_rx) = JobCompletedSender::new_never_used(); + let (_kp_tx, kp_rx) = KillpillSender::new(1); + let token = create_token_for_owner( + &db, + "test-workspace", + "u/test-user", + "", + 100, + "", + &Uuid::nil(), + None, + None, + ) + .await + .unwrap(); + let client = AuthedClient::new( + format!("http://localhost:{port}"), + "test-workspace".to_string(), + token, + None, + ); + + windmill_worker::result_processor::handle_job_error( + &db, + &client, + &MiniCompletedJob::from(step_job), + 0, + None, + windmill_common::error::Error::ExecutionErr( + "simulated worker OOM crash".to_string(), + ), + true, // unrecoverable + Some(&sw_tx), + "", + "test-monitor", + jc_tx, + &kp_rx, + ) + .await; + + // The flow should now run its failure module and complete. + loop { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let done = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + flow_id + ) + .fetch_one(&db) + .await + .unwrap() + .unwrap_or(false); + if done { + break; + } + } + }, + port, + ) + .await; + + completed_job(flow_id, db).await.json_result().unwrap() +} + +/// The hanging-forever first step: only ever completed by the simulated zombie handler. +#[cfg(feature = "deno_core")] +fn hanging_step_value() -> serde_json::Value { + serde_json::json!({ + "input_transforms": {}, + "type": "rawscript", + "language": "deno", + "content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }", + }) +} + +/// The error handler module that marks itself so tests can assert it ran. +#[cfg(feature = "deno_core")] +fn marker_failure_module() -> serde_json::Value { + serde_json::json!({ + "value": { + "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, + "type": "rawscript", + "language": "deno", + "content": "export function main(error) { return { handled_unrecoverable: true, error } }", + } + }) +} + +/// Regression test for WIN-2070: a flow step that fails *unrecoverably* — e.g. its worker was +/// OOM-killed and the failure is surfaced by the zombie job handler — must still trigger the +/// flow's error handler (failure module). Previously, `unrecoverable` failures silently +/// completed the flow with an error and skipped the failure module entirely. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_failure_module_triggered_on_unrecoverable_failure( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [{ "id": "a", "value": hanging_step_value() }], + "failure_module": marker_failure_module(), + })) + .unwrap(); + + let result = run_flow_until_step_running_then_fail_unrecoverably(&db, port, flow).await; + + server.close().await.unwrap(); + + assert_eq!( + result["handled_unrecoverable"], + json!(true), + "failure module (flow error handler) should run for an unrecoverable step failure, got: {result}" + ); + Ok(()) +} + +/// WIN-2070: an unrecoverable failure must NOT be retried even when the step has a retry +/// policy — the original worker is gone, so retrying is pointless. It should fall straight +/// through to the error handler (failure module) instead. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_unrecoverable_failure_skips_retry_runs_failure_module( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [{ + "id": "a", + "value": hanging_step_value(), + "retry": { "constant": { "attempts": 5, "seconds": 0 } }, + }], + "failure_module": marker_failure_module(), + })) + .unwrap(); + + let result = run_flow_until_step_running_then_fail_unrecoverably(&db, port, flow).await; + + server.close().await.unwrap(); + + assert_eq!( + result["handled_unrecoverable"], + json!(true), + "unrecoverable failure should skip retry and run the failure module, got: {result}" + ); + Ok(()) +} + +/// WIN-2070: an unrecoverable failure on a `continue_on_error` step must still route to the +/// error handler rather than silently advancing to the next step (which would hide the worker +/// death). A normal failure on a `continue_on_error` step is tolerated and the flow continues; +/// a worker crash/OOM is not. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Step 'a' hangs (and tolerates failures via continue_on_error); step 'b' would run next + // if the unrecoverable failure were (wrongly) tolerated. + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [ + { + "id": "a", + "value": hanging_step_value(), + "continue_on_error": true, + }, + { + "id": "b", + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "deno", + "content": "export function main() { return { ran_b: true }; }", + }, + }, + ], + "failure_module": marker_failure_module(), + })) + .unwrap(); + + let result = run_flow_until_step_running_then_fail_unrecoverably(&db, port, flow).await; + + server.close().await.unwrap(); + + assert_eq!( + result["handled_unrecoverable"], + json!(true), + "unrecoverable failure on a continue_on_error step should run the failure module, got: {result}" + ); + assert!( + result.get("ran_b").is_none(), + "the step after a continue_on_error step must NOT run on an unrecoverable failure, got: {result}" + ); + Ok(()) +} + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] async fn test_run_wait_result_early_return_with_failure_module( @@ -4997,6 +5252,7 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content: WORKFLOW_AS_CODE.into(), + tag: None, ..RawCode::default() })) .arg("n", json!(3)) diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index d0ede744d3..94d3ef65e0 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -676,6 +676,29 @@ pub fn gemini_event_to_openai_sse_chunks( *tool_call_index += 1; } + // Gemini reports token counts in `usageMetadata` on its final event. Mirror + // OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`, + // empty `choices`) so the frontend's `'usage' in chunk` path records them. + if let Some(usage) = &parsed.usage { + let prompt_tokens = usage.prompt_token_count.unwrap_or(0); + let completion_tokens = usage.candidates_token_count.unwrap_or(0); + let total_tokens = usage + .total_token_count + .unwrap_or(prompt_tokens + completion_tokens); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + } + }); + chunks.push(format!("data: {}\n\n", chunk)); + } + chunks } @@ -836,9 +859,18 @@ fn openai_tool_call_json( mod tests { use super::{ gemini_event_to_openai_sse_chunks, gemini_response_to_openai, parse_gemini_sse_event, - sanitize_schema_for_google, GeminiParsedEvent, GeminiToolCallEvent, + sanitize_schema_for_google, GeminiParsedEvent, GeminiToolCallEvent, GeminiUsageMetadata, }; + /// Strip the `data: ...\n\n` SSE framing and parse the payload as JSON. + fn parse_sse_chunk(chunk: &str) -> serde_json::Value { + let payload = chunk + .strip_prefix("data: ") + .and_then(|c| c.strip_suffix("\n\n")) + .expect("chunk should be wrapped as SSE data"); + serde_json::from_str(payload).expect("chunk should be valid JSON") + } + #[test] fn gemini_response_to_openai_preserves_thought_signature() { let parsed = GeminiParsedEvent { @@ -903,6 +935,94 @@ mod tests { assert_eq!(tool_call["index"], 0); } + #[test] + fn gemini_streaming_emits_usage_chunk() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(12), + candidates_token_count: Some(7), + total_token_count: Some(19), + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + + // Mirrors OpenAI's `stream_options.include_usage`: a terminal chunk with + // top-level `usage` and empty `choices`. + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 12); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 7); + assert_eq!(usage_chunk["usage"]["total_tokens"], 19); + assert_eq!(usage_chunk["choices"], serde_json::json!([])); + } + + #[test] + fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() { + let parsed = GeminiParsedEvent { + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(5), + candidates_token_count: Some(3), + total_token_count: None, + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 5); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 3); + assert_eq!(usage_chunk["usage"]["total_tokens"], 8); + } + + #[test] + fn gemini_streaming_without_usage_emits_no_usage_chunk() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + + assert!( + chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .all(|v| v.get("usage").map(|u| u.is_null()).unwrap_or(true)), + "no usage chunk should be emitted when the event has no usageMetadata" + ); + } + #[test] fn gemini_thought_parts_route_to_reasoning() { let event = r#"{ diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 0c25cd243d..a982b02d35 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; use windmill_common::{ - assets::{AssetKind, AssetUsageKind}, + assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, error::JsonResult, utils::escape_ilike_pattern, @@ -20,6 +20,65 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_assets)) .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) + .route("/graph", get(asset_graph)) + .route("/pipelines", get(list_pipeline_folders)) + .route("/partitions", get(list_partitions)) + .route("/record_materialization", post(record_materialization)) +} + +#[derive(Deserialize)] +struct PartitionsQuery { + // The materialized asset path (`/
`). + path: String, +} + +// Per-partition materialization status for a ducklake asset — drives the +// partition-status grid and the backfill worklist. Materialization targets are +// ducklake-only in v1, so the kind is fixed. +async fn list_partitions( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = windmill_common::materialization::list_materialized_partitions( + &mut *tx, + &w_id, + AssetKind::Ducklake, + &q.path, + ) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +// Record a materialization outcome from a polyglot (Python/TS) `wmill.ducklake` +// helper running as a pipeline step. The DuckDB `// materialize` engine records +// this itself; the SDK helpers post here instead so SDK-materialized slices show +// up in the grid identically. RLS-scoped to the caller's workspace. +async fn record_materialization( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Json(req): Json, +) -> JsonResult<()> { + let mut tx = user_db.begin(&authed).await?; + windmill_common::materialization::record_materialization( + &mut *tx, + &w_id, + req.asset_kind, + &req.asset_path, + &req.partition, + req.status, + req.snapshot_id, + req.row_count, + req.job_id, + req.error.as_deref(), + ) + .await?; + tx.commit().await?; + Ok(Json(())) } #[derive(Deserialize)] @@ -363,3 +422,445 @@ async fn list_favorites( Ok(Json(favorites)) } + +// ------------------------------------------------------------------ +// GET /w/:workspace/assets/graph +// ------------------------------------------------------------------ +// Workspace-wide asset ↔ runnable graph. One row per unique +// (asset_kind, asset_path, usage_kind, usage_path, access_type) — the +// frontend aggregates into nodes and edges. + +#[derive(Deserialize)] +struct GraphQuery { + pub asset_kinds: Option, + pub folder: Option, +} + +#[derive(Serialize, Debug)] +struct GraphAssetNode { + kind: AssetKind, + path: String, +} + +#[derive(Serialize, Debug)] +struct GraphRunnableNode { + path: String, + usage_kind: AssetUsageKind, + // True iff the script was deployed with `// pipeline` — drives the + // pipeline-member visual state on the frontend. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + in_pipeline: bool, +} + +// Lineage edge from parsed r/w usages. One per (runnable, asset, access_type) +// tuple. Informational — not the DAG execution edges. +#[derive(Serialize, Debug)] +struct GraphEdge { + runnable_path: String, + runnable_kind: AssetUsageKind, + asset_kind: AssetKind, + asset_path: String, + access_type: Option, +} + +// Declared `// on ` trigger edge — the actual execution DAG. +// Asset edges come from `script_trigger`; the eight native variants +// (Schedule/Email/Kafka/…/Gcp) come from the per-kind trigger tables joined +// on `script_path`. Each native variant carries just the trigger row's path; +// the config (cron, broker, topic, auth, …) lives in its own UI. +// +// `webhook` is parsed as an annotation marker but has no dedicated trigger +// table — every script gets an implicit webhook endpoint — so no variant +// here. The frontend renders the marker from the source annotations alone. +#[derive(Serialize, Debug)] +#[serde(tag = "trigger_kind", rename_all = "lowercase")] +enum TriggerEdge { + Asset { + asset_kind: AssetKind, + asset_path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Schedule { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Email { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Kafka { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Mqtt { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Nats { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Postgres { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Sqs { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Gcp { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, +} + +#[derive(Serialize, Debug)] +struct AssetGraphResponse { + assets: Vec, + runnables: Vec, + edges: Vec, + triggers: Vec, +} + +async fn asset_graph( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + let kind_filter: Option> = q.asset_kinds.as_ref().map(|s| { + s.split(',') + .filter_map(|k| { + serde_json::from_value::(Value::String(k.trim().into())).ok() + }) + .collect() + }); + let kind_filter_ref = kind_filter.as_deref(); + + let folder_filter = q.folder.as_deref().map(|f| format!("f/{}/%", f)); + + // One row per (asset_kind, asset_path, usage_kind, usage_path, access_type). + // The `usage_kind IN ('script','flow')` clause excludes `job`-kind usage rows + // (runtime-detected, ephemeral) so the graph stays stable. + let rows = sqlx::query!( + r#" + SELECT + asset.kind AS "asset_kind!: AssetKind", + asset.path AS "asset_path!", + asset.usage_kind AS "usage_kind!: AssetUsageKind", + asset.usage_path AS "usage_path!", + asset.usage_access_type::text AS "access_type" + FROM asset + WHERE asset.workspace_id = $1 + AND asset.usage_kind IN ('script', 'flow') + AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2)) + AND ($3::text IS NULL OR asset.usage_path LIKE $3) + GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type + "#, + &w_id, + kind_filter_ref as Option<&[AssetKind]>, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Pipeline asset trigger edges, fetched separately so we can widen the + // runnable_set for trigger-only endpoints (e.g. an asset trigger whose + // asset has no usage in the pipeline yet). Native trigger kinds + // (schedule, kafka, mqtt, …) are *not* in `script_trigger` — they're + // discovered below by querying each native trigger table directly. + let trigger_rows = sqlx::query!( + r#" + SELECT + runnable_kind AS "runnable_kind!: AssetUsageKind", + runnable_path AS "runnable_path!", + trigger_kind::text AS "trigger_kind!", + trigger_ref AS "trigger_ref!" + FROM script_trigger + WHERE workspace_id = $1 + AND trigger_kind = 'asset' + AND ($2::text IS NULL OR runnable_path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Native triggers in scope. Each native trigger table stores its + // single-destination `script_path` directly, so we resolve attachment by + // joining on that field rather than via `script_trigger`. UNION ALL keeps + // it a single round trip; the `kind` column drives the TriggerEdge ctor + // below. `schedule` lives in the `schedule` table, which has its own + // shape (no workspace_id-only filter — it shares `is_flow` like the + // others), but the columns we need line up. + let native_trigger_rows = sqlx::query!( + r#" + SELECT kind, path, script_path, is_flow FROM ( + SELECT 'schedule' AS kind, path, script_path, is_flow FROM schedule + WHERE workspace_id = $1 + AND script_path IS NOT NULL + UNION ALL + SELECT 'email', path, script_path, is_flow FROM email_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'kafka', path, script_path, is_flow FROM kafka_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'nats', path, script_path, is_flow FROM nats_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'postgres', path, script_path, is_flow FROM postgres_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'sqs', path, script_path, is_flow FROM sqs_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'gcp', path, script_path, is_flow FROM gcp_trigger + WHERE workspace_id = $1 + ) t + WHERE ($2::text IS NULL OR script_path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Which scripts in scope are pipeline members (have `// pipeline`). + let pipeline_member_paths = sqlx::query!( + r#" + SELECT path AS "path!" + FROM script + WHERE workspace_id = $1 + AND auto_kind = 'pipeline' + AND archived = false + AND deleted = false + AND ($2::text IS NULL OR path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Existing scripts / flows in the workspace. Used to filter out + // orphan trigger rows whose `script_path` no longer resolves — those + // would otherwise be added to `runnable_set` below and surface as + // phantom "deployed" runnables on the canvas (matching what the user + // can deploy a new trigger against: nothing). + let existing_script_paths = sqlx::query_scalar!( + r#"SELECT path AS "path!" FROM script + WHERE workspace_id = $1 + AND archived = false + AND deleted = false"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + let existing_flow_paths = sqlx::query_scalar!( + r#"SELECT path AS "path!" FROM flow WHERE workspace_id = $1 AND archived = false"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + let pipeline_member_script_paths: std::collections::HashSet = + pipeline_member_paths.into_iter().map(|r| r.path).collect(); + let existing_script_paths: std::collections::HashSet = + existing_script_paths.into_iter().collect(); + let existing_flow_paths: std::collections::HashSet = + existing_flow_paths.into_iter().collect(); + let runnable_exists = |kind: AssetUsageKind, path: &str| match kind { + AssetUsageKind::Script => existing_script_paths.contains(path), + AssetUsageKind::Flow => existing_flow_paths.contains(path), + // `Job` is a runtime-detected ephemeral runnable (asset usage rows + // only), never a target of a stored trigger row. Treat as existing + // so we don't accidentally drop ephemeral lineage edges. + AssetUsageKind::Job => true, + }; + + let mut edges = Vec::with_capacity(rows.len()); + let mut asset_set: std::collections::HashSet<(AssetKind, String)> = Default::default(); + let mut runnable_set: std::collections::HashSet<(AssetUsageKind, String)> = Default::default(); + + // Every pipeline member in scope goes into the graph, even when the parser + // didn't detect any asset r/w and the script has no triggers yet. Without + // this, a freshly-saved pipeline script whose template body hasn't been + // filled in would vanish from the pipeline view on graph refetch. + for path in &pipeline_member_script_paths { + runnable_set.insert((AssetUsageKind::Script, path.clone())); + } + + for r in rows { + // Drop asset usage rows whose runnable target was archived/deleted + // but whose row in `asset` is still around — those would otherwise + // surface as a phantom "deployed" runnable on the canvas with no + // way to interact with it, since the underlying script/flow no + // longer exists. + if !runnable_exists(r.usage_kind, &r.usage_path) { + continue; + } + asset_set.insert((r.asset_kind, r.asset_path.clone())); + runnable_set.insert((r.usage_kind, r.usage_path.clone())); + edges.push(GraphEdge { + runnable_path: r.usage_path, + runnable_kind: r.usage_kind, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + access_type: r.access_type, + }); + } + + let mut triggers: Vec = + Vec::with_capacity(trigger_rows.len() + native_trigger_rows.len()); + for t in trigger_rows { + // Drop orphan asset-trigger rows — their target runnable no longer + // exists (script/flow archived or deleted, or was never deployed). + // Without this, an orphan row would surface as a phantom "deployed" + // runnable on the canvas (no `unsaved` flag, can't actually be + // run / re-targeted by a new trigger). + if !runnable_exists(t.runnable_kind, &t.runnable_path) { + continue; + } + runnable_set.insert((t.runnable_kind, t.runnable_path.clone())); + if t.trigger_kind.as_str() == "asset" { + // trigger_ref is `` — parse back out so both + // endpoints match what the frontend uses for node ids. + if let Some((asset_kind, asset_path)) = parse_asset_trigger_ref(&t.trigger_ref) { + // Make sure the source asset has a node even if nothing + // reads/writes it in this folder. + asset_set.insert((asset_kind, asset_path.clone())); + triggers.push(TriggerEdge::Asset { + asset_kind, + asset_path, + runnable_kind: t.runnable_kind, + runnable_path: t.runnable_path, + }); + } + } + // Native kinds (schedule, kafka, mqtt, …) come from per-kind trigger + // tables below. + } + + // Native trigger attachments — one TriggerEdge per row, the kind chosen + // from the discriminator. Add the runnable to the set so a script with + // no asset edges but a kafka/schedule attachment still renders on the + // canvas. + for t in native_trigger_rows { + let kind = t.kind.unwrap_or_default(); + let path = t.path.unwrap_or_default(); + let script_path = t.script_path.unwrap_or_default(); + let runnable_kind = if t.is_flow.unwrap_or(false) { + AssetUsageKind::Flow + } else { + AssetUsageKind::Script + }; + // Same orphan filter as the asset-trigger loop above — drop trigger + // rows whose target script/flow no longer exists so the graph + // doesn't synthesize a phantom deployed runnable. + if !runnable_exists(runnable_kind, &script_path) { + continue; + } + runnable_set.insert((runnable_kind, script_path.clone())); + let edge = match kind.as_str() { + "schedule" => TriggerEdge::Schedule { path, runnable_kind, runnable_path: script_path }, + "email" => TriggerEdge::Email { path, runnable_kind, runnable_path: script_path }, + "kafka" => TriggerEdge::Kafka { path, runnable_kind, runnable_path: script_path }, + "mqtt" => TriggerEdge::Mqtt { path, runnable_kind, runnable_path: script_path }, + "nats" => TriggerEdge::Nats { path, runnable_kind, runnable_path: script_path }, + "postgres" => TriggerEdge::Postgres { path, runnable_kind, runnable_path: script_path }, + "sqs" => TriggerEdge::Sqs { path, runnable_kind, runnable_path: script_path }, + "gcp" => TriggerEdge::Gcp { path, runnable_kind, runnable_path: script_path }, + _ => continue, + }; + triggers.push(edge); + } + + let mut assets: Vec = asset_set + .into_iter() + .map(|(kind, path)| GraphAssetNode { kind, path }) + .collect(); + assets.sort_by(|a, b| a.path.cmp(&b.path)); + + let mut runnables: Vec = runnable_set + .into_iter() + .map(|(usage_kind, path)| { + let in_pipeline = usage_kind == AssetUsageKind::Script + && pipeline_member_script_paths.contains(&path); + GraphRunnableNode { path, usage_kind, in_pipeline } + }) + .collect(); + runnables.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(Json(AssetGraphResponse { + assets, + runnables, + edges, + triggers, + })) +} + +// ------------------------------------------------------------------ +// GET /w/:workspace/assets/pipelines +// ------------------------------------------------------------------ +// Distinct folder names that contain at least one pipeline-member script +// (auto_kind='pipeline'). Used by the pipeline-editor folder picker and +// the "Pipeline" entry in folder views. Keyed by the partial index on +// `script (workspace_id, path) WHERE auto_kind='pipeline' ...` so this +// is effectively O(matches). + +#[derive(Serialize, Debug)] +struct PipelineFolder { + folder: String, + script_count: i64, +} + +async fn list_pipeline_folders( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#" + SELECT + substring(path from '^f/([^/]+)/') AS "folder!", + COUNT(*) AS "script_count!" + FROM script + WHERE workspace_id = $1 + AND auto_kind = 'pipeline' + AND archived = false + AND deleted = false + AND path LIKE 'f/%' + GROUP BY substring(path from '^f/([^/]+)/') + ORDER BY substring(path from '^f/([^/]+)/') + "#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| PipelineFolder { folder: r.folder, script_count: r.script_count }) + .collect(), + )) +} diff --git a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs index a0f7f7d460..88ffe6bb26 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs @@ -1336,3 +1336,188 @@ async fn test_compare_workspaces_fork_only_folder_visibility( Ok(()) } + +/// Regression test: deleting a fork must purge its `workspace_diff` and +/// `skip_workspace_diff_tally` rows. These tables are keyed by workspace id with +/// no FK cascade, and a fork id is reused when a fork is deleted and recreated +/// under the same name. If the cached diff rows survive the delete, they leak +/// onto the next fork sharing that id and produce a spurious "changes not +/// visible" warning that hides the deploy button (WIN-2066). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_delete_fork_purges_workspace_diff(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + let base_url = format!("http://localhost:{port}/api"); + + // Create the fork so the caller owns it (delete is authorized for fork owners). + let fork_response = client + .client() + .post(&format!( + "{base_url}/w/test-workspace/workspaces/create_fork" + )) + .json(&json!({ + "id": "wm-fork-test-workspace", + "name": "Test Fork", + "color": "#0000ff" + })) + .send() + .await?; + assert!( + fork_response.status().is_success(), + "Fork creation should succeed: {}", + fork_response.status() + ); + + // Seed cached diff state for the fork: as the fork side of a pair, as the + // source side of a pair, and a skip-tally row. + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork) + VALUES ('test-workspace', 'wm-fork-test-workspace', 'f/shared/leaky', 'script', 1, 0, true, true, true)" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes) + VALUES ('wm-fork-test-workspace', 'test-workspace', 'f/shared/other', 'script', 0, 1, true)" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO skip_workspace_diff_tally (workspace_id) VALUES ('wm-fork-test-workspace')" + ) + .execute(&db) + .await?; + + // Delete the fork through the real handler. + let delete_response = client + .client() + .delete(&format!("{base_url}/workspaces/delete/wm-fork-test-workspace")) + .send() + .await?; + assert!( + delete_response.status().is_success(), + "Fork deletion should succeed: {}", + delete_response.status() + ); + + let leftover_diffs = sqlx::query_scalar!( + "SELECT COUNT(*) FROM workspace_diff + WHERE source_workspace_id = 'wm-fork-test-workspace' + OR fork_workspace_id = 'wm-fork-test-workspace'" + ) + .fetch_one(&db) + .await?; + assert_eq!( + leftover_diffs, + Some(0), + "workspace_diff rows referencing the deleted fork must be purged" + ); + + let leftover_skip = sqlx::query_scalar!( + "SELECT COUNT(*) FROM skip_workspace_diff_tally WHERE workspace_id = 'wm-fork-test-workspace'" + ) + .fetch_one(&db) + .await?; + assert_eq!( + leftover_skip, + Some(0), + "skip_workspace_diff_tally row for the deleted fork must be purged" + ); + + Ok(()) +} + +/// Regression test: creating a fork must start with clean diff state even when +/// the (reusable) fork id was previously occupied by a deleted fork. Stale +/// `workspace_diff` / `skip_workspace_diff_tally` rows left behind by an earlier +/// occupant would otherwise leak onto the new fork — a stale skip row suppresses +/// comparison entirely, and stale diff rows produce a spurious "changes not +/// visible" warning that hides the deploy button (WIN-2066). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_create_fork_purges_stale_diff_state(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + let base_url = format!("http://localhost:{port}/api"); + + // Simulate leftovers from a previously deleted fork that reused this id: + // diff rows on both sides plus a skip-tally row, with no workspace yet. + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork) + VALUES ('test-workspace', 'wm-fork-test-workspace', 'f/shared/leaky', 'script', 1, 0, true, true, true)" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes) + VALUES ('wm-fork-test-workspace', 'test-workspace', 'f/shared/other', 'script', 0, 1, true)" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO skip_workspace_diff_tally (workspace_id) VALUES ('wm-fork-test-workspace')" + ) + .execute(&db) + .await?; + + // Create the fork reusing that id; the conflict check passes because no + // workspace row exists for it. + let fork_response = client + .client() + .post(&format!( + "{base_url}/w/test-workspace/workspaces/create_fork" + )) + .json(&json!({ + "id": "wm-fork-test-workspace", + "name": "Test Fork", + "color": "#0000ff" + })) + .send() + .await?; + assert!( + fork_response.status().is_success(), + "Fork creation should succeed: {}", + fork_response.status() + ); + + let leftover_diffs = sqlx::query_scalar!( + "SELECT COUNT(*) FROM workspace_diff + WHERE source_workspace_id = 'wm-fork-test-workspace' + OR fork_workspace_id = 'wm-fork-test-workspace'" + ) + .fetch_one(&db) + .await?; + assert_eq!( + leftover_diffs, + Some(0), + "stale workspace_diff rows must be purged on fork creation" + ); + + let leftover_skip = sqlx::query_scalar!( + "SELECT COUNT(*) FROM skip_workspace_diff_tally WHERE workspace_id = 'wm-fork-test-workspace'" + ) + .fetch_one(&db) + .await?; + assert_eq!( + leftover_skip, + Some(0), + "stale skip_workspace_diff_tally row must be purged on fork creation" + ); + + Ok(()) +} diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 5ea3088837..ecf0ff0b61 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] enterprise = ["windmill-common/enterprise"] private = ["windmill-common/private", "windmill-dep-map/private"] -python = ["dep:windmill-parser-py"] +python = ["dep:windmill-parser-py", "dep:windmill-parser-py-asset"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] [dependencies] windmill-common = { workspace = true, default-features = false } @@ -23,6 +23,11 @@ windmill-audit.workspace = true windmill-git-sync.workspace = true windmill-dep-map.workspace = true windmill-parser-ts.workspace = true +windmill-parser.workspace = true +windmill-parser-ts-asset.workspace = true +windmill-parser-sql-asset.workspace = true +windmill-parser-sql.workspace = true +windmill-parser-yaml.workspace = true axum.workspace = true futures.workspace = true @@ -41,3 +46,4 @@ lazy_static.workspace = true tokio.workspace = true prometheus = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } +windmill-parser-py-asset = { workspace = true, optional = true } diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs new file mode 100644 index 0000000000..5debb0d271 --- /dev/null +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -0,0 +1,254 @@ +//! Server-side asset inference at script deploy. +//! +//! The `asset` rows written at deploy drive the asset-trigger cascade +//! (`fetch_producer_writes` in windmill-queue). Historically they came solely +//! from the client-supplied `NewScript.assets`, so a client with broken +//! inference (e.g. a failed wasm load) deploying `assets: null` silently +//! killed the producer side of the cascade while the subscriber side (parsed +//! server-side from `// on` annotations) kept looking wired. This module makes +//! asset *presence* server-authoritative by re-parsing the deployed content +//! with the same parsers the frontend wasm builds wrap. +//! +//! Merge semantics are a union: server-parsed assets are always present; +//! client entries are kept too (they may carry `alt_access_type` — the user's +//! manual access-type override — or come from client-side detection paths the +//! server has no parser for). For duplicate `(kind, path)` keys the server's +//! parser-derived fields win and the client's `alt_access_type` is preserved. +//! Languages without a server parser (or whose parse fails) fall back to the +//! client list unchanged, matching the previous behavior. + +use std::collections::BTreeMap; + +use windmill_common::{ + assets::{ + asset_access_type_from_parser, asset_kind_from_parser, AssetKind, AssetUsageAccessType, + AssetWithAltAccessType, + }, + scripts::ScriptLang, +}; + +/// Mirror of the frontend `inferAssets` language dispatch (infer.ts): only +/// these languages have a body-asset parser. Returns `None` for unsupported +/// languages or on parse failure — callers then keep the client-supplied list. +fn parse_assets_for_lang( + lang: &ScriptLang, + content: &str, +) -> Option> { + let parsed = match lang { + ScriptLang::DuckDb => windmill_parser_sql_asset::parse_assets(content), + ScriptLang::Bun | ScriptLang::Deno | ScriptLang::Nativets => { + windmill_parser_ts_asset::parse_assets(content) + } + #[cfg(feature = "python")] + ScriptLang::Python3 => windmill_parser_py_asset::parse_assets(content), + ScriptLang::Ansible => windmill_parser_yaml::parse_assets(content), + _ => return None, + }; + match parsed { + Ok(out) => Some(out.assets), + Err(e) => { + tracing::warn!( + "server-side asset inference failed for a {} script; falling back to \ + client-supplied assets: {e:#}", + lang.as_str() + ); + None + } + } +} + +/// Mirror of the frontend `getCommentPrefix` (infer.ts) — the languages whose +/// leading comment block is scanned for `volume: ` annotations. +fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby + | ScriptLang::Rlang => Some("#"), + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => Some("//"), + _ => None, + } +} + +/// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): ` +/// volume: ` lines in the leading comment block, each an `rw` volume +/// asset. Scanning stops at the first non-comment line (blank lines are +/// skipped), exactly like the frontend. +fn parse_volume_annotations(content: &str, prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let Some(after) = trimmed.strip_prefix(prefix) else { + break; + }; + let after = after.trim(); + if let Some(rest) = after.strip_prefix("volume:") { + if let Some(path) = rest.trim().split_whitespace().next() { + volumes.push(AssetWithAltAccessType { + path: path.to_string(), + kind: AssetKind::Volume, + access_type: Some(AssetUsageAccessType::RW), + alt_access_type: None, + columns: None, + }); + } + } + } + volumes +} + +/// Parse the deployed content into asset usages, mirroring the frontend's +/// `inferAssets` (body parser per language + volume annotations). `None` +/// means "no server parser produced anything for this language" — distinct +/// from `Some(vec![])`, which is an authoritative "this script uses no +/// assets". +fn infer_script_assets(lang: &ScriptLang, content: &str) -> Option> { + let body_assets = parse_assets_for_lang(lang, content); + let volume_assets = comment_prefix(lang) + .map(|p| parse_volume_annotations(content, p)) + .unwrap_or_default(); + if body_assets.is_none() && volume_assets.is_empty() { + return None; + } + let mut out: Vec = body_assets + .unwrap_or_default() + .into_iter() + .map(|a| AssetWithAltAccessType { + path: a.path, + kind: asset_kind_from_parser(a.kind), + access_type: a.access_type.map(asset_access_type_from_parser), + alt_access_type: None, + columns: a.columns.map(|cols| { + cols.into_iter() + .map(|(k, v)| (k, asset_access_type_from_parser(v))) + .collect::>() + }), + }) + .collect(); + out.extend(volume_assets); + Some(out) +} + +/// The asset list to persist at deploy: server-parsed assets unioned with the +/// client-supplied ones. See the module docs for the exact semantics. +pub fn effective_script_assets( + lang: &ScriptLang, + content: &str, + client_assets: Option>, +) -> Option> { + let Some(inferred) = infer_script_assets(lang, content) else { + return client_assets; + }; + let mut merged: Vec = inferred; + for client in client_assets.into_iter().flatten() { + if let Some(existing) = merged + .iter_mut() + .find(|a| a.kind == client.kind && a.path == client.path) + { + // Server parse wins for parser-derived fields; the client's + // alt_access_type is the user's manual override — keep it. + if existing.alt_access_type.is_none() { + existing.alt_access_type = client.alt_access_type; + } + } else { + merged.push(client); + } + } + Some(merged) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset( + kind: AssetKind, + path: &str, + access: Option, + ) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind, + access_type: access, + alt_access_type: None, + columns: None, + } + } + + // The bug class this module exists for: client deploys with no assets, + // but the content demonstrably writes one — the server parse must + // produce the producer row anyway. + #[test] + fn duckdb_write_survives_empty_client_assets() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS\n\ + SELECT * FROM (SELECT 1 AS placeholder);"; + let got = effective_script_assets(&ScriptLang::DuckDb, content, None).unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].kind, AssetKind::DataTable); + assert_eq!(got[0].path, "main/out_table"); + assert_eq!(got[0].access_type, Some(AssetUsageAccessType::W)); + } + + #[test] + fn client_alt_access_type_is_preserved_on_match() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS SELECT 1;"; + let mut client = asset(AssetKind::DataTable, "main/out_table", None); + client.alt_access_type = Some(AssetUsageAccessType::RW); + let got = + effective_script_assets(&ScriptLang::DuckDb, content, Some(vec![client])).unwrap(); + assert_eq!(got.len(), 1); + // Parser-derived access wins; the user's alt override rides along. + assert_eq!(got[0].access_type, Some(AssetUsageAccessType::W)); + assert_eq!(got[0].alt_access_type, Some(AssetUsageAccessType::RW)); + } + + #[test] + fn client_only_entries_are_kept() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS SELECT 1;"; + let extra = asset( + AssetKind::S3Object, + "bucket/file.parquet", + Some(AssetUsageAccessType::R), + ); + let got = effective_script_assets(&ScriptLang::DuckDb, content, Some(vec![extra])).unwrap(); + assert_eq!(got.len(), 2); + assert!(got.iter().any(|a| a.kind == AssetKind::S3Object)); + } + + #[test] + fn unsupported_language_falls_back_to_client() { + let client = vec![asset( + AssetKind::S3Object, + "b/f.json", + Some(AssetUsageAccessType::W), + )]; + let got = effective_script_assets(&ScriptLang::Go, "package main", Some(client.clone())); + assert_eq!(got.map(|v| v.len()), Some(1)); + assert_eq!( + effective_script_assets(&ScriptLang::Go, "package main", None).is_none(), + true + ); + } + + #[test] + fn volume_annotations_parsed_from_leading_comment_block() { + let content = "// volume: my_vol\n// some other comment\nconsole.log(1)\n// volume: ignored_after_code\n"; + let got = effective_script_assets(&ScriptLang::Bun, content, None).unwrap(); + let vols: Vec<_> = got.iter().filter(|a| a.kind == AssetKind::Volume).collect(); + assert_eq!(vols.len(), 1); + assert_eq!(vols[0].path, "my_vol"); + assert_eq!(vols[0].access_type, Some(AssetUsageAccessType::RW)); + } +} diff --git a/backend/windmill-api-scripts/src/lib.rs b/backend/windmill-api-scripts/src/lib.rs index 9bc8318fc4..3ac3764e96 100644 --- a/backend/windmill-api-scripts/src/lib.rs +++ b/backend/windmill-api-scripts/src/lib.rs @@ -6,4 +6,5 @@ * LICENSE-AGPL for a copy of the license. */ +pub mod asset_inference; pub mod scripts; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 79583835b0..ea54b83c31 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -44,8 +44,9 @@ use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ assets::{ - clear_static_asset_usage, clear_static_asset_usage_by_script_hash, - insert_static_asset_usage, AssetUsageKind, + clear_script_triggers, clear_static_asset_usage, clear_static_asset_usage_by_script_hash, + insert_script_trigger, parse_duration_secs, parse_pipeline_annotations, + replace_static_asset_usage, trigger_spec_to_row, AssetUsageKind, TriggerSpec, }, error::{self, to_anyhow}, min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2}, @@ -1235,7 +1236,149 @@ async fn create_script_internal<'c>( let ci_test_refs = windmill_common::schema::parse_ci_test_annotation(&ns.content, &lang.as_comment_lit()); - let auto_kind = if ci_test_refs.is_some() { + // `pipeline` wins over `test` and any client-supplied auto_kind. The + // bare `// pipeline` marker is the opt-in signal for pipeline + // membership; parsed writes tell us what is produced (we don't record + // them in auto_kind itself). + let pipeline_annotations = parse_pipeline_annotations(&ns.content); + // `// freshness` is parsed but enforcement is a not-yet-implemented + // enterprise feature (skeleton in windmill_common::pipeline_advanced). + // Surface a clear TODO at deploy rather than silently accepting an + // annotation that does nothing. + if pipeline_annotations.freshness.is_some() { + tracing::warn!( + "{}", + windmill_common::pipeline_advanced::freshness_enforcement_todo() + ); + } + // `// materialize` materializes a `ducklake:///
` target from a + // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake + // target would otherwise deploy, register a producer in the asset graph, then + // silently no-op at run time (`build_materialized_query` returns `Ok(None)`), + // and a non-DuckDB script never reaches the executor that records state. The + // managed-only checks (single trailing SELECT, no SQL args) come after — a + // `manual` script owns its DDL and skips them. + if let Some(m) = pipeline_annotations.materialize.as_ref() { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// materialize` is only supported for DuckDB scripts, not {}. Use the \ + wmll.ducklake helpers to materialize from other languages.", + ns.language.as_str() + ))); + } + if m.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + return Err(Error::BadRequest( + "`// materialize` only supports a DuckLake target \ + (`ducklake:///
`); other asset kinds aren't materializable." + .to_string(), + )); + } + if !m.target_path.contains('/') { + return Err(Error::BadRequest(format!( + "`// materialize` needs a table in the target: \ + `ducklake://{0}/
` (got `ducklake://{0}`).", + m.target_path + ))); + } + if !m.manual { + if let Err(e) = windmill_parser::sql_materialize::classify_wrap(&ns.content) { + return Err(Error::BadRequest(e.message())); + } + // Managed materialize strips line comments when it wraps the SELECT, + // so a `-- $name (TYPE)` declaration is lost while its `$name` + // reference survives in the embedded SELECT — it would run unbound. + // Managed materialize takes no SQL args (the partition is supplied by + // the engine, not bound). Reject declared args with a clear error. + if let Ok(sig) = windmill_parser_sql::parse_duckdb_sig(&ns.content) { + if !sig.args.is_empty() { + let names = sig + .args + .iter() + .map(|a| format!("${}", a.name)) + .collect::>() + .join(", "); + return Err(Error::BadRequest(format!( + "managed `// materialize` cannot take SQL arguments ({names}): wrapping your \ + SELECT drops the `-- $arg` declarations, so they would run unbound. The \ + partition is supplied by the engine — reference its value with the \ + `{{partition}}` token, or use `// materialize manual` to write the DDL (and \ + bind args) yourself." + ))); + } + } + } + // `key=` (merge) and `append` are mutually exclusive reconciliation + // strategies; append (INSERT-only) wins. Surface the conflict rather + // than silently dropping the dedup the author may have intended. + if m.unique_key.is_some() && m.append { + tracing::warn!( + "script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)", + ns.path + ); + } + } + let in_pipeline = pipeline_annotations.in_pipeline; + // `// trigger all` → AND join barrier (else OR, the default). + let pipeline_join_all = !pipeline_annotations.join_mode.is_any(); + // Script-level `// debounce ` default; a per-`// on debounce=` + // overrides it (precedence resolved per edge below). + let pipeline_debounce_default = pipeline_annotations.debounce_default; + let pipeline_triggers = pipeline_annotations.triggers; + // `// tag ` overrides the caller-supplied tag at deploy. Source + // wins, matching the wipe-and-reinsert convention of other pipeline + // annotations. Applied before the dep-job tag selection below (which + // special-cases dedicated_worker / bunnative / `$args[`) so that path + // sees the annotation-overridden value. + if let Some(t) = pipeline_annotations.tag.clone() { + ns.tag = Some(t); + } + // `// retry []` is PARSED but PARKED: a retried subscriber + // is wrapped in a SingleStepFlow, whose run is a flow step and therefore + // ineligible for asset dispatch (asset_dispatch::is_eligible_kind) — so a + // retried subscriber would silently become a cascade dead-end (P1). We do + // not persist it to script_trigger; the cascade ignores retry until this + // is fixed. TODO(pipeline-retry): re-enable once cascade dispatch handles + // flow-wrapped producers. + if pipeline_annotations.retry.is_some() { + tracing::warn!( + "`// retry` on {} is not yet supported in the asset cascade and is ignored \ + (a retried subscriber cannot trigger its downstream). TODO(pipeline-retry).", + ns.path + ); + } + // Asset presence is server-authoritative: re-parse the deployed content + // (same parsers the frontend wasm wraps) and union with the client list. + // The `asset` rows written below drive the asset-trigger cascade, so a + // client deploying `assets: null` (e.g. broken wasm inference) must not + // silently kill the producer side while `// on` subscribers stay wired. + let effective_assets = crate::asset_inference::effective_script_assets( + &ns.language, + &ns.content, + ns.assets.take(), + ); + // Register the `// materialize` target as a write asset so the deployed + // asset graph shows this script as the producer of the managed table — the + // body's `SELECT` doesn't express the write (the runtime generates it), so + // server-side inference wouldn't otherwise link it. + let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() { + let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind); + let mut a = effective_assets.unwrap_or_default(); + if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) { + a.push(windmill_common::assets::AssetWithAltAccessType { + path: m.target_path.clone(), + kind, + access_type: Some(windmill_common::assets::AssetUsageAccessType::W), + alt_access_type: None, + columns: None, + }); + } + Some(a) + } else { + effective_assets + }; + let auto_kind = if in_pipeline { + Some("pipeline".to_string()) + } else if ci_test_refs.is_some() { Some("test".to_string()) } else { auto_kind @@ -1312,7 +1455,9 @@ async fn create_script_internal<'c>( &authed, ), validate_schema, - ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), + effective_assets + .as_ref() + .and_then(|a| serde_json::to_value(a).ok()), guarded_debounce_key, guarded_debounce_delay_s, ns.cache_ignore_s3_path, @@ -1550,11 +1695,69 @@ async fn create_script_internal<'c>( ); } - clear_static_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in ns.assets.as_ref().into_iter().flatten() { - insert_static_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script) - .await?; + // Clear + reinsert this script's producer rows at script_path (== ns.path), + // invalidating the producer-writes cache once iff the write-producer set + // changed (see replace_static_asset_usage). + replace_static_asset_usage( + &mut tx, + &w_id, + &script_path, + effective_assets.as_deref().unwrap_or(&[]), + ) + .await?; + + // Pipeline trigger edges: wipe-and-reinsert per deploy so removing an + // `// on ...` annotation drops the edge. Only Asset / Schedule produce + // a row — native trigger marker annotations (`// on kafka`, etc.) are + // discovered by the graph endpoint directly from the per-kind trigger + // tables, so `trigger_spec_to_row` returns None for those. + clear_script_triggers(&mut *tx, &w_id, &ns.path, AssetUsageKind::Script).await?; + // On rename, also drop the OLD path's trigger rows. clear is keyed by + // path (no by-hash variant), and only `ns.path` is wiped above — without + // this, stale `// on` edges for the old path keep matching producers and + // would trigger a script later recreated at that path even if it has no + // annotation (P1). (Producer/asset rows for the old path are already + // cleared via clear_static_asset_usage_by_script_hash on the parent.) + if let Some(ref old) = p_path_opt { + if old != &ns.path { + clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; + } } + for spec in &pipeline_triggers { + let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { + continue; + }; + // Effective debounce for this edge: per-`// on debounce=` wins, + // else the script-level `// debounce` default. Debounce only + // applies to asset-cascade edges; other trigger kinds get none. + let debounce_s = match spec { + TriggerSpec::Asset { debounce: Some(d), .. } => parse_duration_secs(d), + TriggerSpec::Asset { .. } => pipeline_debounce_default + .as_deref() + .and_then(parse_duration_secs), + _ => None, + }; + insert_script_trigger( + &mut *tx, + &w_id, + AssetUsageKind::Script, + &ns.path, + trigger_kind, + &trigger_ref, + pipeline_join_all, + debounce_s, + // retry parked — see TODO(pipeline-retry) above. + None, + None, + ) + .await?; + } + + // Schedule annotations (`// on schedule`) are marker-only — the binding + // lives on the schedule row's own `script_path` field, which the user + // creates separately via the schedule editor. No script-create-time + // reconciliation is needed (and there are no "managed" schedules to + // upsert/delete anymore). let permissioned_as = username_to_permissioned_as(&authed.username); if let Some(parent_hash) = ns.parent_hash { @@ -2548,6 +2751,10 @@ async fn archive_script_by_path( .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + // Pipeline event hygiene: an archived script must not be triggered by + // anything. Wipe declared `// on ...` edges (asset-event subscribers + // look these up). + clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2625,6 +2832,9 @@ async fn archive_script_by_hash( check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + // Pipeline event hygiene: archived scripts must not be triggered by + // anything. Wipe declared `// on ...` edges. + clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2685,6 +2895,10 @@ async fn delete_script_by_hash( check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + // Pipeline event hygiene: a deleted script must not be triggered by + // anything. Wipe declared `// on ...` edges. Idempotent — safe even if + // the script was never a pipeline member. + clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2797,6 +3011,11 @@ async fn delete_script_by_path( .execute(&mut *tx) .await?; + // Pipeline event hygiene: a deleted script must not be triggered by + // anything. Wipe declared `// on ...` edges. Idempotent — safe even if + // the script was never a pipeline member. + clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + if !query.keep_captures.unwrap_or(false) { sqlx::query!( "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 9fd1091d66..7a278afc49 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -249,13 +249,15 @@ use windmill_object_store::build_object_store_from_settings; #[cfg(feature = "parquet")] pub async fn test_s3_bucket( - _authed: ApiAuthed, + authed: ApiAuthed, Extension(db): Extension, Json(test_s3_bucket): Json, ) -> error::Result { use bytes::Bytes; use futures::StreamExt; + require_super_admin(&db, &authed.email).await?; + let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) .await? .store; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 52c5aa3d72..bf0824f974 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -48,7 +48,9 @@ use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; use windmill_common::{ error::{Error, JsonResult, Result}, - global_settings::AUTOMATE_USERNAME_CREATION_SETTING, + global_settings::{ + AUTOMATE_USERNAME_CREATION_SETTING, DISABLE_WORKSPACE_INVITE_EMAILS_SETTING, + }, oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, utils::{paginate, rd_string, require_admin, Pagination}, }; @@ -4774,6 +4776,30 @@ async fn check_fork_w_id_conflict(db: &DB, w_id: &str) -> Result<()> { } } +/// A fork id is reusable: it is freed when a fork is deleted and can be claimed +/// again under the same name. `workspace_diff` and `skip_workspace_diff_tally` +/// are keyed by workspace id with no FK cascade, so a freshly created fork could +/// inherit cached diff state from a previous occupant of its id — a stale skip +/// row suppresses comparison entirely, and stale workspace_diff rows produce a +/// spurious "changes not visible" warning that hides the deploy button. Clear +/// both so a new fork always starts with clean diff state, regardless of how the +/// id was freed. +async fn purge_stale_fork_diff_state(db: &DB, fork_id: &str) -> Result<()> { + sqlx::query!( + "DELETE FROM workspace_diff WHERE source_workspace_id = $1 OR fork_workspace_id = $1", + fork_id + ) + .execute(db) + .await?; + sqlx::query!( + "DELETE FROM skip_workspace_diff_tally WHERE workspace_id = $1", + fork_id + ) + .execute(db) + .await?; + Ok(()) +} + lazy_static::lazy_static! { pub static ref CREATE_WORKSPACE_REQUIRE_SUPERADMIN: bool = { @@ -5977,6 +6003,7 @@ async fn create_workspace_fork_branch( // Fail before creating any git branch so a name conflict doesn't leave a // dangling branch on the synced repos. check_fork_w_id_conflict(&db, &nw.id).await?; + purge_stale_fork_diff_state(&db, &nw.id).await?; Ok(Json( handle_fork_branch_creation(&authed.email, &authed.username, &db, &w_id, &nw.id).await?, @@ -6121,6 +6148,7 @@ async fn create_workspace_fork( // re-using a taken (possibly archived) fork id reports the actual // conflict instead of a misleading "maximum number of workspaces" error. check_fork_w_id_conflict(&db, &nw.id).await?; + purge_stale_fork_diff_state(&db, &nw.id).await?; #[cfg(not(feature = "enterprise"))] _check_nb_of_workspaces(&db).await?; @@ -6441,6 +6469,20 @@ async fn unarchive_workspace( Ok(format!("Unarchived workspace {}", &w_id)) } +/// Whether the instance is configured to suppress the email notifications sent +/// when a user is invited or added to a workspace. Defaults to false (emails on). +async fn workspace_invite_emails_disabled(db: &DB) -> Result { + Ok( + windmill_common::global_settings::load_value_from_global_settings( + db, + DISABLE_WORKSPACE_INVITE_EMAILS_SETTING, + ) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false), + ) +} + async fn invite_user( ApiAuthed { username, is_admin, .. }: ApiAuthed, Extension(db): Extension, @@ -6499,16 +6541,18 @@ async fn invite_user( tx.commit().await?; - send_email_if_possible( - &format!("Invited to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} + if !workspace_invite_emails_disabled(&db).await? { + send_email_if_possible( + &format!("Invited to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - (**BASE_URL.load()).clone() - ), - &nu.email, - ); + (**BASE_URL.load()).clone() + ), + &nu.email, + ); + } webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace { email: nu.email.clone(), @@ -6651,17 +6695,19 @@ async fn add_user( ) .await?; - send_email_if_possible( - &format!("Added to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} by {} + if !workspace_invite_emails_disabled(&db).await? { + send_email_if_possible( + &format!("Added to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} by {} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - authed.email, - (**BASE_URL.load()).clone() - ), - &nu.email, - ); + authed.email, + (**BASE_URL.load()).clone() + ), + &nu.email, + ); + } webhook.send_instance_event(InstanceEvent::UserAddedWorkspace { workspace: w_id.clone(), diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 4c6b57172f..aa64514b4c 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -868,6 +868,25 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + // workspace_diff and skip_workspace_diff_tally are keyed by workspace id with no + // FK cascade. A fork id is reused when a fork is deleted and recreated under the + // same name, so leaving these rows behind leaks the previous fork's cached diff + // verdicts onto the new fork — causing a spurious "changes not visible" warning + // that hides the deploy button. + sqlx::query!( + "DELETE FROM workspace_diff WHERE source_workspace_id = $1 OR fork_workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM skip_workspace_diff_tally WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 0cf2e9aca0..135131af41 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -1,7 +1,7 @@ { "openapi": "3.0.3", "info": { - "version": "1.723.0", + "version": "1.728.0", "title": "Windmill API", "contact": { "name": "Windmill Team", @@ -9700,9 +9700,9 @@ "type": "string", "description": "OAuth client secret for resource-level credentials (client_credentials flow only)" }, - "cc_token_url": { + "cc_instance": { "type": "string", - "description": "OAuth token URL override for resource-level authentication (client_credentials flow only)" + "description": "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side (client_credentials flow only). The token URL is never caller-supplied." }, "mcp_server_url": { "type": "string", @@ -9739,7 +9739,7 @@ } } }, - "/oauth/connect_client_credentials/{client}": { + "/w/{workspace}/oauth/connect_client_credentials/{client}": { "post": { "summary": "connect OAuth using client credentials", "operationId": "connectClientCredentials", @@ -9747,6 +9747,9 @@ "oauth" ], "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, { "name": "client", "in": "path", @@ -9773,21 +9776,17 @@ }, "cc_client_id": { "type": "string", - "description": "OAuth client ID for resource-level authentication" + "description": "OAuth client ID. Omit to use the credentials configured on the provider's instance OAuth entry." }, "cc_client_secret": { "type": "string", - "description": "OAuth client secret for resource-level authentication" + "description": "OAuth client secret. Omit to use the credentials configured on the provider's instance OAuth entry." }, - "cc_token_url": { + "cc_instance": { "type": "string", - "description": "OAuth token URL override for resource-level authentication" + "description": "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side. The token URL is never caller-supplied." } - }, - "required": [ - "cc_client_id", - "cc_client_secret" - ] + } } } } @@ -10000,7 +9999,23 @@ "schema": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "supports_client_credentials": { + "type": "boolean" + }, + "has_shared_credentials": { + "type": "boolean" + } + }, + "required": [ + "name", + "supports_client_credentials", + "has_shared_credentials" + ] } } } @@ -10049,6 +10064,10 @@ "items": { "type": "string" } + }, + "client_credentials_configured": { + "type": "boolean", + "description": "The instance OAuth entry carries shared client-credentials, so the connect dialog can skip the bring-your-own form and run the exchange server-side" } } } @@ -12191,10 +12210,18 @@ "type": "string", "description": "Best-effort, read from the draft JSON's `summary` field when the editor shape carries one." }, + "draft_path": { + "type": "string", + "description": "User-typed friendly path from the draft JSON's `draft_path`, when set and different from the storage path (e.g. a never-deployed item parked at `u/{user}/draft_{uuid}`)." + }, "draft_only": { "type": "boolean", "description": "No deployed counterpart exists at this path — the draft is the whole item." }, + "legacy_draft": { + "type": "boolean", + "description": "The listed draft is a legacy workspace-level row (email NULL) predating the per-user drafts migration. Only true when no per-user draft exists at this path." + }, "created_at": { "type": "string", "format": "date-time" @@ -12204,6 +12231,7 @@ "kind", "path", "draft_only", + "legacy_draft", "created_at" ] } @@ -12316,6 +12344,10 @@ "force": { "type": "boolean", "description": "Skip the conflict check and overwrite the server copy." + }, + "legacy": { + "type": "boolean", + "description": "Delete-only. Target the legacy workspace-level row (email NULL) instead of the current user's row. Used to discard a legacy draft from the review page." } } } @@ -12359,9 +12391,10 @@ "description": "Creates a new script when the path does not already exist.\nCreates a new version of an existing script when called with the same path and the current `parent_hash`.\n", "operationId": "createScript", "x-mcp-tool": true, - "x-mcp-instructions": "To create a script, specify the path (e.g., 'f/my_folder/my_script'), the content (source code), and the language. For TypeScript, use 'bun' unless deno-specific APIs are needed.", + "x-mcp-instructions": "To create a NEW script, specify the path (e.g., 'f/my_folder/my_script'), the content (source code), and the language, and leave parent_hash unset. For TypeScript, use 'bun' unless deno-specific APIs are needed. To UPDATE an existing script, do NOT delete and recreate it: call this tool with the same path and set parent_hash to the script's current hash, which you can read from the `hash` field returned by getScriptByPath. This creates a new version while preserving the script's history.", "x-mcp-tool-include-fields": [ "path", + "parent_hash", "content", "language", "summary", @@ -33609,8 +33642,16 @@ "type": "string", "nullable": true, "description": "Workspace username of the draft owner. `null` represents\nthe legacy workspace-level (NULL-email) row. Emails never\nleave the server.\n" + }, + "draft_saved_at": { + "type": "string", + "format": "date-time", + "description": "When this user's draft was last saved (`draft.created_at`),\nsurfaced in the fork modal as \"Last updated\".\n" } - } + }, + "required": [ + "draft_saved_at" + ] } } }, diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 22792b38e3..e45131d81b 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.723.0 + version: 1.728.0 title: Windmill API contact: name: Windmill Team @@ -7696,6 +7696,16 @@ paths: Emails never leave the server. + draft_saved_at: + type: string + format: date-time + description: > + When this user's draft was last saved + (`draft.created_at`), + + surfaced in the fork modal as "Last updated". + required: + - draft_saved_at required: &ref_79 - is_draft /w/{workspace}/variables/get_value/{path}: @@ -8900,11 +8910,14 @@ paths: description: >- OAuth client secret for resource-level credentials (client_credentials flow only) - cc_token_url: + cc_instance: type: string description: >- - OAuth token URL override for resource-level authentication - (client_credentials flow only) + Instance name for built-in providers whose + client-credentials token URL is instance-templated; + substituted into the fixed-host registry template + server-side (client_credentials flow only). The token URL is + never caller-supplied. mcp_server_url: type: string description: MCP server URL for MCP OAuth token refresh @@ -8926,13 +8939,17 @@ paths: text/plain: schema: type: string - /oauth/connect_client_credentials/{client}: + /w/{workspace}/oauth/connect_client_credentials/{client}: post: summary: connect OAuth using client credentials operationId: connectClientCredentials tags: - oauth parameters: + - name: workspace + in: path + required: true + schema: *ref_4 - name: client in: path description: OAuth client name @@ -8953,16 +8970,21 @@ paths: type: string cc_client_id: type: string - description: OAuth client ID for resource-level authentication + description: >- + OAuth client ID. Omit to use the credentials configured on + the provider's instance OAuth entry. cc_client_secret: type: string - description: OAuth client secret for resource-level authentication - cc_token_url: + description: >- + OAuth client secret. Omit to use the credentials configured + on the provider's instance OAuth entry. + cc_instance: type: string - description: OAuth token URL override for resource-level authentication - required: - - cc_client_id - - cc_client_secret + description: >- + Instance name for built-in providers whose + client-credentials token URL is instance-templated; + substituted into the fixed-host registry template + server-side. The token URL is never caller-supplied. responses: '200': description: OAuth token response @@ -9113,7 +9135,18 @@ paths: schema: type: array items: - type: string + type: object + properties: + name: + type: string + supports_client_credentials: + type: boolean + has_shared_credentials: + type: boolean + required: + - name + - supports_client_credentials + - has_shared_credentials /oauth/get_connect/{client}: get: summary: get oauth connect @@ -9145,6 +9178,12 @@ paths: type: array items: type: string + client_credentials_configured: + type: boolean + description: >- + The instance OAuth entry carries shared + client-credentials, so the connect dialog can skip the + bring-your-own form and run the exchange server-side /teams/activities: post: summary: send update to Microsoft Teams activity @@ -12719,11 +12758,24 @@ paths: description: >- Best-effort, read from the draft JSON's `summary` field when the editor shape carries one. + draft_path: + type: string + description: >- + User-typed friendly path from the draft JSON's + `draft_path`, when set and different from the storage + path (e.g. a never-deployed item parked at + `u/{user}/draft_{uuid}`). draft_only: type: boolean description: >- No deployed counterpart exists at this path — the draft is the whole item. + legacy_draft: + type: boolean + description: >- + The listed draft is a legacy workspace-level row (email + NULL) predating the per-user drafts migration. Only true + when no per-user draft exists at this path. created_at: type: string format: date-time @@ -12731,6 +12783,7 @@ paths: - kind - path - draft_only + - legacy_draft - created_at /w/{workspace}/drafts/get/{kind}/{path}: get: @@ -12832,6 +12885,12 @@ paths: force: type: boolean description: Skip the conflict check and overwrite the server copy. + legacy: + type: boolean + description: >- + Delete-only. Target the legacy workspace-level row (email + NULL) instead of the current user's row. Used to discard a + legacy draft from the review page. responses: '200': description: save result @@ -12862,11 +12921,17 @@ paths: operationId: createScript x-mcp-tool: true x-mcp-instructions: >- - To create a script, specify the path (e.g., 'f/my_folder/my_script'), - the content (source code), and the language. For TypeScript, use 'bun' - unless deno-specific APIs are needed. + To create a NEW script, specify the path (e.g., + 'f/my_folder/my_script'), the content (source code), and the language, + and leave parent_hash unset. For TypeScript, use 'bun' unless + deno-specific APIs are needed. To UPDATE an existing script, do NOT + delete and recreate it: call this tool with the same path and set + parent_hash to the script's current hash, which you can read from the + `hash` field returned by getScriptByPath. This creates a new version + while preserving the script's history. x-mcp-tool-include-fields: - path + - parent_hash - content - language - summary diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a988480fc6..458055b1dd 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.727.0 + version: 1.734.0 title: Windmill API contact: @@ -6617,9 +6617,9 @@ paths: cc_client_secret: type: string description: "OAuth client secret for resource-level credentials (client_credentials flow only)" - cc_token_url: + cc_instance: type: string - description: "OAuth token URL override for resource-level authentication (client_credentials flow only)" + description: "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side (client_credentials flow only). The token URL is never caller-supplied." mcp_server_url: type: string description: "MCP server URL for MCP OAuth token refresh" @@ -6640,13 +6640,14 @@ paths: schema: type: string - /oauth/connect_client_credentials/{client}: + /w/{workspace}/oauth/connect_client_credentials/{client}: post: summary: connect OAuth using client credentials operationId: connectClientCredentials tags: - oauth parameters: + - $ref: "#/components/parameters/WorkspaceId" - name: client in: path description: OAuth client name @@ -6667,16 +6668,13 @@ paths: type: string cc_client_id: type: string - description: "OAuth client ID for resource-level authentication" + description: "OAuth client ID. Omit to use the credentials configured on the provider's instance OAuth entry." cc_client_secret: type: string - description: "OAuth client secret for resource-level authentication" - cc_token_url: + description: "OAuth client secret. Omit to use the credentials configured on the provider's instance OAuth entry." + cc_instance: type: string - description: "OAuth token URL override for resource-level authentication" - required: - - cc_client_id - - cc_client_secret + description: "Instance name for built-in providers whose client-credentials token URL is instance-templated; substituted into the fixed-host registry template server-side. The token URL is never caller-supplied." responses: "200": description: OAuth token response @@ -6810,7 +6808,18 @@ paths: schema: type: array items: - type: string + type: object + properties: + name: + type: string + supports_client_credentials: + type: boolean + has_shared_credentials: + type: boolean + required: + - name + - supports_client_credentials + - has_shared_credentials /oauth/get_connect/{client}: get: @@ -6843,6 +6852,9 @@ paths: type: array items: type: string + client_credentials_configured: + type: boolean + description: "The instance OAuth entry carries shared client-credentials, so the connect dialog can skip the bring-your-own form and run the exchange server-side" /teams/activities: post: @@ -8227,6 +8239,11 @@ paths: - draft parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: all_users + in: query + description: List every draft in the workspace (all users), not just the current user's own + legacy rows. Other users' rows come back with `mine=false` (view-only). + schema: + type: boolean responses: "200": description: the user's drafts @@ -8256,7 +8273,25 @@ paths: created_at: type: string format: date-time - required: [kind, path, draft_only, legacy_draft, created_at] + can_write: + type: boolean + description: Whether the current user may deploy/discard this draft (same check the deploy/discard endpoints enforce). + mine: + type: boolean + description: The row belongs to the current user (own draft or the legacy no-owner row) and is therefore actionable. Always true in the default listing; with `all_users=true`, other users' rows are false (view-only). + draft_users: + description: | + Draft authors at this (path, kind) — the legacy NULL-email row surfaced as a null username. + Populated only for the shared full-page-editor kinds (script/flow/app/raw_app); omitted for + drawer kinds, which keep their drafts private. Feeds the Draft badge's owner-avatar circles. + type: array + items: + type: object + properties: + username: + type: string + nullable: true + required: [kind, path, draft_only, legacy_draft, created_at, can_write, mine] /w/{workspace}/drafts/get/{kind}/{path}: get: @@ -8294,6 +8329,35 @@ paths: "404": description: no draft for that owner at that path + /w/{workspace}/drafts/get_own/{kind}/{path}: + get: + summary: fetch the current user's own draft content at a path (any kind) + operationId: getOwnDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: the user's draft content, or null when none exists + content: + application/json: + schema: + nullable: true + type: object + properties: + value: {} + created_at: + type: string + format: date-time + required: [value, created_at] + /w/{workspace}/drafts/update/{kind}/{path}: post: summary: upsert (or clear) the current user's draft at a path @@ -8328,6 +8392,10 @@ paths: legacy: type: boolean description: Delete-only. Target the legacy workspace-level row (email NULL) instead of the current user's row. Used to discard a legacy draft from the review page. + created_at: + type: string + format: date-time + description: Upsert-only override for the stored creation timestamp. Normal saves omit it (stamped server-side); the localStorage→DB migration passes the draft's original write time so migrated drafts keep their age. responses: "200": description: save result @@ -8344,6 +8412,41 @@ paths: format: date-time required: [status, current_timestamp] + /w/{workspace}/drafts/migrate_legacy/{kind}/{path}: + post: + summary: resolve a legacy (workspace-level) draft (admin only) + description: Delete a legacy draft (email NULL) or assign it to the authed admin as a per-user draft. Workspace admins / superadmins only. + operationId: migrateLegacyDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + action: + type: string + enum: [delete, assign_to_self] + description: delete the legacy draft, or take ownership of it. + required: [action] + responses: + "200": + description: migration result + content: + text/plain: + schema: + type: string + /w/{workspace}/scripts/create: post: summary: create script @@ -13237,6 +13340,147 @@ paths: required: - created_at + /w/{workspace}/jobs_u/dispatch_events/{id}: + get: + summary: list asset-trigger dispatch events for a producer job + description: > + Returns the chronological log of decisions the asset-trigger + dispatcher made after this producer job completed. Each row is one + (subscriber, asset write) decision: `dispatched` (with + `child_job_id`), `join_pending` (with `received_inputs` / + `required_inputs` / `partition`), or `skipped` (with `reason`). + Rows are reaped automatically when the producer's `v2_job` row is + deleted by the retention sweep. + operationId: listDispatchEvents + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: dispatch events for this producer job + content: + application/json: + schema: + type: array + items: + type: object + properties: + subscriber_path: + type: string + asset_kind: + type: string + enum: + - s3object + - resource + - variable + - ducklake + - datatable + - volume + asset_path: + type: string + outcome: + type: string + enum: + - dispatched + - join_pending + - skipped + child_job_id: + type: string + format: uuid + partition: + type: string + received_inputs: + type: integer + required_inputs: + type: integer + debounce_s: + type: integer + reason: + type: string + created_at: + type: string + format: date-time + required: + - subscriber_path + - asset_kind + - asset_path + - outcome + - created_at + + /w/{workspace}/jobs/asset_dispatch_edges: + get: + summary: list asset-cascade producer→child job edges for a folder + description: > + Returns the `dispatched` asset-trigger edges (producer job → child job) + whose subscriber lives under `path_start`. Lets a pipeline view + reconstruct the cascade tree of a folder by job id and group connected + runs. Visibility follows the producer job's RLS. + operationId: listAssetDispatchEdges + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path_start + in: query + required: true + description: Folder path prefix the children live under, e.g. `f/orders/`. + schema: + type: string + - name: created_after + in: query + required: false + description: Only edges dispatched at/after this instant. + schema: + type: string + format: date-time + responses: + "200": + description: asset-cascade edges for the folder + content: + application/json: + schema: + type: array + items: + type: object + properties: + producer_job_id: + type: string + format: uuid + child_job_id: + type: string + format: uuid + description: Set for `dispatched`; absent for `join_pending` inputs. + subscriber_path: + type: string + outcome: + type: string + enum: + - dispatched + - join_pending + asset_kind: + type: string + enum: + - s3object + - resource + - variable + - ducklake + - datatable + - volume + asset_path: + type: string + created_at: + type: string + format: date-time + required: + - producer_job_id + - subscriber_path + - outcome + - asset_kind + - asset_path + - created_at + /w/{workspace}/jobs/completed/delete/{id}: post: summary: delete completed job (erase content but keep run id) @@ -13683,6 +13927,12 @@ paths: type: integer approver: type: string + view_token: + type: string + description: >- + Share-read-link token for the flow. An authenticated workspace + member can append it as a `view_token` query param on the run + page to read a flow they don't otherwise have access to. /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}: get: @@ -13934,6 +14184,13 @@ paths: required: - resume_id - approver + view_token: + type: string + description: >- + Share-read-link token for the parent flow. An authenticated + workspace member can append it as a `view_token` query param + on the run page to read a flow they don't otherwise have + access to. required: - job - approvers @@ -20853,6 +21110,149 @@ paths: type: string description: The asset path + /w/{workspace}/assets/graph: + get: + summary: Get the workspace-wide asset <-> runnable graph + operationId: getAssetsGraph + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_kinds + in: query + description: Filter by asset kinds (comma-separated list) + schema: + type: string + - name: folder + in: query + description: Scope the graph to runnables in a single folder + schema: + type: string + responses: + "200": + description: asset graph nodes, lineage edges and trigger edges + content: + application/json: + schema: + type: object + required: [assets, runnables, edges, triggers] + properties: + assets: + type: array + items: + type: object + required: [kind, path] + properties: + kind: + $ref: "#/components/schemas/AssetKind" + path: + type: string + runnables: + type: array + items: + type: object + required: [path, usage_kind] + properties: + path: + type: string + usage_kind: + $ref: "#/components/schemas/AssetUsageKind" + in_pipeline: + type: boolean + description: True iff the script is a pipeline member (deployed with `// pipeline`). Omitted when false. + edges: + type: array + items: + type: object + required: + [runnable_path, runnable_kind, asset_kind, asset_path] + properties: + runnable_path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + access_type: + $ref: "#/components/schemas/AssetUsageAccessType" + triggers: + type: array + items: + oneOf: + - type: object + description: Asset trigger edge (`// on `) + required: + [ + trigger_kind, + asset_kind, + asset_path, + runnable_kind, + runnable_path, + ] + properties: + trigger_kind: + type: string + enum: [asset] + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + runnable_path: + type: string + - type: object + description: Native trigger edge (schedule, email, kafka, ...). `path` is the trigger row's path. + required: + [trigger_kind, path, runnable_kind, runnable_path] + properties: + trigger_kind: + type: string + enum: + - schedule + - email + - kafka + - mqtt + - nats + - postgres + - sqs + - gcp + path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + runnable_path: + type: string + + /w/{workspace}/assets/pipelines: + get: + summary: List folders that contain at least one pipeline-member script + operationId: listPipelineFolders + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: folders containing pipeline scripts, with their script counts + content: + application/json: + schema: + type: array + items: + type: object + required: [folder, script_count] + properties: + folder: + type: string + description: The folder name (without the `f/` prefix) + script_count: + type: integer + format: int64 + description: Number of pipeline-member scripts in the folder + /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -21587,6 +21987,7 @@ components: - trigger_nextcloud - trigger_google - trigger_github + - data_pipeline # Do not change next line. It is used by python-client for pre-processing # -- INLINE START -- OpenFlow: @@ -22504,6 +22905,13 @@ components: type: string parent_hash: type: string + auto_parent: + type: boolean + description: >- + When true, the backend resolves the parent to the current deployed + head for this path within the transaction (ignoring parent_hash), + instead of failing with a "lineage must be linear" error when the + supplied parent_hash is stale. summary: type: string description: @@ -24749,6 +25157,7 @@ components: - azure - google - github + - asset TriggerMode: description: job trigger mode diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 3cb2a1aef6..0615735081 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -397,10 +397,13 @@ async fn list_apps( "draft.path IS NOT NULL as is_draft", // Per-path draft owners as a JSON array; see scripts.rs for the rationale // (admins-workspace identity fallback, legacy NULL-email row). + // `app`/`raw_app` are separate draft kinds over one `app` table — match + // either (like the `is_draft` join below), else a deployed raw app's draft + // owners are dropped and the row shows "Draft" with no user badge. "(SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END)) ORDER BY COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ - WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ = 'app') as draft_users", + WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) .left() diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index c8ed841e19..7b8d468da8 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -282,6 +282,12 @@ pub async fn migrate( 20260207000002, 20260207000003, 20260207000004, + // Squashed pre-release pipeline migrations: the per-column ALTERs on + // script_trigger and the dispatch_event subscriber index were folded + // back into these two CREATEs, changing their checksum. Both are + // idempotent, so re-applying on an already-migrated DB is a no-op. + 20260423050000, + 20260523055641, ]; for m in migrator.migrations.iter() { if m.migration_type.is_down_migration() { diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index c6146fb47a..9d4976489b 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -9,7 +9,7 @@ use crate::db::{ApiAuthed, DB}; use axum::{ - extract::{Extension, Path}, + extract::{Extension, Path, Query}, routing::{get, post}, Json, Router, }; @@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize}; use windmill_common::{ db::UserDB, error::{Error, Result}, - user_drafts::{UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, + user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, variables::{build_crypt, encrypt}, }; @@ -25,7 +25,9 @@ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_drafts)) .route("/get/{kind}/{*path}", get(get_draft_for_user)) + .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } #[derive(Serialize, sqlx::FromRow)] @@ -50,6 +52,30 @@ pub struct DraftListItem { /// row exists at this (path, kind) — the DISTINCT ON prefers an owned row. pub legacy_draft: bool, pub created_at: chrono::DateTime, + /// All draft authors at this `(path, kind)`, for the shared full-page-editor + /// kinds (script/flow/app/raw_app) only — feeds the home-page-style owner + /// circles on the review page. `None` for drawer kinds, which keep their + /// drafts private. + #[serde(skip_serializing_if = "Option::is_none")] + pub draft_users: Option>>, + /// Whether the authed user may deploy/discard this draft — the same check + /// the deploy/discard endpoints enforce. Computed per row after the query, + /// so it defaults to `false` when read from the row. + #[sqlx(default)] + pub can_write: bool, + /// The listed row belongs to the authed user (own draft or the legacy + /// no-owner row) and is therefore actionable by them. Always `true` in the + /// default (own-drafts) listing; only meaningful with `all_users=true`, + /// where other users' rows surface as `false` (view-only — you can't deploy + /// someone else's draft). + pub mine: bool, +} + +#[derive(Deserialize)] +pub struct ListDraftsQuery { + /// List every draft in the workspace (all users), not just the authed + /// user's own + legacy rows. Other users' rows come back with `mine=false`. + pub all_users: Option, } /// Every draft the authed user has in this workspace, across all kinds — the @@ -59,7 +85,9 @@ pub struct DraftListItem { async fn list_drafts( authed: ApiAuthed, Extension(db): Extension, + Extension(user_db): Extension, Path(w_id): Path, + Query(query): Query, ) -> Result>> { // Operators have no drafts of their own (they can't write any, see // `require_can_write_path`), so this list is always empty for them. They @@ -67,20 +95,58 @@ async fn list_drafts( if authed.is_operator { return Ok(Json(vec![])); } - let rows = sqlx::query_as::<_, DraftListItem>(&list_drafts_query()) + let all_users = query.all_users.unwrap_or(false); + let rows = sqlx::query_as::<_, DraftListItem>(&list_drafts_query(all_users)) .bind(&w_id) .bind(&authed.email) .fetch_all(&db) .await?; - Ok(Json(rows)) + // Per-row permission gating: + // - own drafts (incl. legacy no-owner rows, `mine = true`): the actionable + // gate is write permission — run the exact check deploy/discard enforce so + // the UI never offers an action that would 403. + // - other users' drafts (only present with `all_users`, `mine = false`): the + // UI never lets you act on them (`isSelectable` requires `mine`), so skip + // the write probe (`can_write = false`) and instead require READ access — + // otherwise the broadened listing would disclose the path/summary/authors + // of items the caller can't see. Unreadable rows are dropped, mirroring the + // `require_can_read_path` gate on `/drafts/get`. + let mut out = Vec::with_capacity(rows.len()); + for mut row in rows { + if row.mine { + row.can_write = + match require_can_write_path(&authed, &db, &user_db, &w_id, row.kind, &row.path) + .await + { + Ok(()) => true, + Err(Error::NotAuthorized(_)) => false, + Err(e) => return Err(e), + }; + out.push(row); + } else { + // `require_can_read_path` denies with `NotFound` (it hides existence) + // and, for some paths, `NotAuthorized` — both mean "not visible to the + // caller", so drop the row. Any other error is a real failure. + match require_can_read_path(&authed, &user_db, &w_id, row.kind, &row.path).await { + Ok(()) => { + row.can_write = false; + out.push(row); + } + Err(Error::NotFound(_)) | Err(Error::NotAuthorized(_)) => {} + Err(e) => return Err(e), + } + } + } + Ok(Json(out)) } /// Build the `list_drafts` SQL, generating the `draft_only` CASE from /// `deployed_table()` (shared single source — can't drift from the access /// check). Table names come from the closed enum, never user input. Kinds /// with no path-keyed table get no arm and fall to `ELSE true`. -/// `$1` = workspace_id, `$2` = email. -fn list_drafts_query() -> String { +/// `$1` = workspace_id, `$2` = email. With `all_users` the owner filter is +/// dropped so every workspace draft is listed (others' rows get `mine=false`). +fn list_drafts_query(all_users: bool) -> String { let mut case = String::from("CASE d.typ::text\n"); for kind in UserDraftItemKind::ALL { let Some(table) = kind.deployed_table() else { @@ -101,15 +167,35 @@ fn list_drafts_query() -> String { )); } case.push_str(" ELSE true\nEND"); - // `(d.email = $2 OR d.email IS NULL)` lists the user's own drafts AND the - // legacy NULL-email rows; `DISTINCT ON (d.path, d.typ)` with `email IS NULL` - // last collapses a (path, kind) that has both to the owned row. + // Owner circles, mirroring the home-page list subquery (see apps.rs): every + // draft author at this (path, kind), legacy NULL-email row surfaced as a + // null username. Restricted to the shared full-page-editor kinds — drawer + // kinds keep their drafts private, so we never reveal their authors. + let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( + SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN du.workspace_id = 'admins' THEN du.email END)) + ORDER BY COALESCE(u.username, CASE WHEN du.workspace_id = 'admins' THEN du.email END) NULLS LAST) + FROM draft du + LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email + WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ + ) ELSE NULL END"#; + // Default lists the user's own drafts AND the legacy NULL-email rows; with + // `all_users` the filter is dropped to list every workspace draft. + let owner_filter = if all_users { + "" + } else { + " AND (d.email = $2 OR d.email IS NULL)" + }; + // `DISTINCT ON (d.path, d.typ)` keeps one row per item; the ORDER BY + // priority below picks the user's own row first, then the legacy NULL row, + // then (only with `all_users`) another user's row. `mine`/`legacy_draft` + // describe that kept row. format!( r#"SELECT DISTINCT ON (d.path, d.typ) d.path, d.typ AS kind, d.created_at, d.value ->> 'summary' AS summary, + {draft_users} AS draft_users, -- Friendly typed path, by kind (mirrors the home-page list -- endpoints): scripts bind the Path widget to `script.path`, -- so it round-trips through the draft JSON's own `path`; @@ -124,10 +210,12 @@ fn list_drafts_query() -> String { d.path ) AS draft_path, (d.email IS NULL) AS legacy_draft, + (d.email = $2 OR d.email IS NULL) AS mine, {case} AS draft_only FROM draft d - WHERE d.workspace_id = $1 AND (d.email = $2 OR d.email IS NULL) - ORDER BY d.path, d.typ, (d.email IS NULL)"# + WHERE d.workspace_id = $1{owner_filter} + ORDER BY d.path, d.typ, + CASE WHEN d.email = $2 THEN 0 WHEN d.email IS NULL THEN 1 ELSE 2 END"# ) } @@ -152,6 +240,12 @@ pub struct SaveDraftRequest { /// the email-scoped delete otherwise can't reach. #[serde(default)] pub legacy: bool, + /// Upsert-only override for the stored `created_at`. Normal saves omit it + /// and the row is stamped `now()`; the localStorage→DB migration passes the + /// draft's original write time (or epoch 0 when unknown) so migrated drafts + /// keep their age instead of all resurfacing to the top as freshly created. + #[serde(default)] + pub created_at: Option>, } #[derive(Serialize, Debug)] @@ -182,7 +276,17 @@ async fn update_draft( ) -> Result> { let email = &authed.email; let path = path.to_path(); - require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + // Saving a draft requires write permission on the underlying path. Deleting + // (discarding) one's OWN draft does not: the email-scoped row belongs to the + // authed user, so they can always discard it even after losing write access + // to the underlying item (e.g. a draft-only item whose folder perms changed). + // The DELETE below is scoped to `email = authed.email`, so it can only ever + // touch the caller's own row. Legacy (NULL-email) rows aren't owned by anyone + // — they keep the write gate. + let is_own_discard = req.value.is_none() && !req.legacy; + if !is_own_discard { + require_can_write_path(&authed, &db, &user_db, &w_id, kind, path).await?; + } let applied_at = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext @@ -192,13 +296,19 @@ async fn update_draft( } else { serde_json::to_string(value).unwrap() }; + // `draft.value` is a `json` column, so a U+0000 (NUL) would persist as an + // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. + // Strip it here so a NUL never reaches the column. + let serialized = strip_json_nul(serialized); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. + // `created_at` defaults to `now()` but the migration overrides it ($8) + // so a migrated draft keeps its original age instead of jumping to top. sqlx::query_scalar!( r#"INSERT INTO draft (workspace_id, email, path, typ, value, created_at) - VALUES ($1, $2, $3, $4, $5::text::json, now()) + VALUES ($1, $2, $3, $4, $5::text::json, COALESCE($8::timestamptz, now())) ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL - DO UPDATE SET value = EXCLUDED.value, created_at = now() + DO UPDATE SET value = EXCLUDED.value, created_at = EXCLUDED.created_at WHERE $7::bool = true OR $6::timestamptz IS NULL OR draft.created_at <= $6::timestamptz @@ -210,6 +320,7 @@ async fn update_draft( serialized, req.last_sync, req.force, + req.created_at, ) .fetch_optional(&db) .await? @@ -281,6 +392,128 @@ async fn update_draft( } } +#[derive(Deserialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum MigrateLegacyDraftAction { + /// Discard the legacy row entirely. + Delete, + /// Move the legacy row's content onto the authed admin's own row, then + /// drop the legacy row — so it becomes a normal per-user draft. + AssignToSelf, +} + +#[derive(Deserialize, Debug)] +pub struct MigrateLegacyDraftRequest { + pub action: MigrateLegacyDraftAction, +} + +/// Resolve a LEGACY (workspace-level, `email IS NULL`) draft. These predate the +/// per-user drafts migration and have no owner, so only workspace admins (and +/// superadmins, which carry `is_admin` in a workspace) may delete one or claim +/// it as their own. +async fn migrate_legacy_draft( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Json(req): Json, +) -> Result { + if !authed.is_admin { + return Err(Error::NotAuthorized( + "only workspace admins can migrate legacy drafts".to_string(), + )); + } + let path = path.to_path(); + match req.action { + MigrateLegacyDraftAction::Delete => { + sqlx::query!( + r#"DELETE FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL"#, + &w_id, + path, + kind as UserDraftItemKind, + ) + .execute(&db) + .await?; + Ok(format!("Deleted legacy draft at {path}")) + } + MigrateLegacyDraftAction::AssignToSelf => { + // Take ownership: move the legacy value onto the admin's own row + // (replacing any existing own draft) and drop the legacy row, in one + // statement. `ON CONFLICT` matches the partial unique index that + // covers `email IS NOT NULL`. + let moved = sqlx::query_scalar!( + r#"WITH legacy AS ( + DELETE FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email IS NULL + RETURNING value + ) + INSERT INTO draft (workspace_id, email, path, typ, value, created_at) + SELECT $1, $4, $2, $3, value, now() FROM legacy + ON CONFLICT (workspace_id, path, typ, email) WHERE email IS NOT NULL + DO UPDATE SET value = EXCLUDED.value, created_at = now() + RETURNING 1 as "one!""#, + &w_id, + path, + kind as UserDraftItemKind, + &authed.email, + ) + .fetch_optional(&db) + .await?; + if moved.is_none() { + return Err(Error::NotFound(format!("no legacy draft at {path}"))); + } + Ok(format!("Assigned legacy draft at {path} to you")) + } + } +} + +/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to +/// store in the `json`-typed `draft.value` (a NUL there would later make any +/// `->>`/`to_jsonb` extraction raise `22P05`). +/// +/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a +/// backslash only ever occurs inside a string, so one backslash-parity-aware +/// pass removes every real NUL escape — covering values and keys alike — while +/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal +/// text `u0000`) intact. O(n) over the bytes with no `serde_json::Value` tree to +/// allocate, and the fast path (no such substring at all) returns the input +/// untouched. The slow path is reached not only by genuinely poisoned values but +/// by any value that legitimately contains `u0000` after a backslash (e.g. script +/// source), so it must stay allocation-light for potentially large drafts. +fn strip_json_nul(serialized: String) -> String { + if !serialized.contains("\\u0000") { + return serialized; + } + let bytes = serialized.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] != b'\\' { + out.push(bytes[i]); + i += 1; + continue; + } + // Consume the whole run of backslashes. An even run is N/2 escaped + // backslashes and leaves the next char unescaped; an odd run ends in an + // escaping backslash, so a following `u0000` is a real NUL escape. + let run_start = i; + while i < bytes.len() && bytes[i] == b'\\' { + i += 1; + } + let run = i - run_start; + if run % 2 == 1 && bytes[i..].starts_with(b"u0000") { + // Drop the escaping backslash + `u0000`; keep the leading literal pairs. + out.extend(std::iter::repeat(b'\\').take(run - 1)); + i += 5; + } else { + out.extend(std::iter::repeat(b'\\').take(run)); + } + } + // Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain + // valid UTF-8 (and valid JSON). + String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8") +} + /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -393,6 +626,37 @@ async fn get_draft_for_user( }) } +/// Fetch the AUTHED user's OWN draft at a path, for any kind — including +/// private kinds (`shares_drafts_across_users() == false`). Backs editors with +/// no deployed-item GET to overlay a draft onto: the `data_pipeline` bundle is +/// keyed at a folder path with no runnable to hang `get_draft` on, so it loads +/// its in-flight state from here. Returns `null` (200) when the user has no +/// draft there, so a fresh pipeline isn't a 404. Secret-variable values come +/// back `$encrypted:`-prefixed, same as `get_draft_for_user` — variable editors +/// use their own overlay GET, not this route. +async fn get_own_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, +) -> Result>> { + let path = path.to_path(); + require_can_read_path(&authed, &user_db, &w_id, kind, path).await?; + let row = sqlx::query_as!( + DraftForUser, + r#"SELECT value as "value!: sqlx::types::Json>", created_at + FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4"#, + &w_id, + path, + kind as UserDraftItemKind, + &authed.email, + ) + .fetch_optional(&db) + .await?; + Ok(Json(row)) +} + /// The deployed table RLS resolves item-level `extra_perms` against. /// Delegates to `UserDraftItemKind::deployed_table()` (the shared single /// source); `None` kinds fall through to the path-only access check. @@ -540,3 +804,64 @@ async fn require_can_read_path( } Err(Error::NotFound(format!("no draft visible at {path}"))) } + + +#[cfg(test)] +mod tests { + use super::strip_json_nul; + + // Parse the (NUL-free) result so assertions read clearly. + fn parsed(s: String) -> serde_json::Value { + serde_json::from_str(&s).expect("strip_json_nul must return valid JSON") + } + + #[test] + fn clean_value_is_returned_byte_for_byte() { + let s = r#"{"summary":"all good","n":1}"#.to_string(); + assert_eq!(strip_json_nul(s.clone()), s); + } + + #[test] + fn real_nul_in_value_is_stripped() { + let out = strip_json_nul(r#"{"summary":"hi\u0000there"}"#.to_string()); + assert!(!out.contains(r"\u0000")); + assert_eq!(parsed(out)["summary"], "hithere"); + } + + #[test] + fn legit_escaped_backslash_is_a_noop() { + // JSON "a\\u0000b" decodes to the 8-char string a,backslash,u,0,0,0,0,b + // — not a NUL — so the value is already clean and round-trips byte-for-byte. + let s = r#"{"summary":"a\\u0000b"}"#.to_string(); + assert_eq!(strip_json_nul(s.clone()), s); + } + + #[test] + fn collision_real_and_literal_both_handled() { + // "a" carries a real NUL; "b" carries the literal text backslash-u0000. + // The value walk strips the former and leaves the latter intact — the + // pathological case that needed a fallback in SQL is trivial in Rust. + let v = parsed(strip_json_nul(r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string())); + assert_eq!(v["a"], "xy"); + assert_eq!(v["b"], "p\\u0000q"); + } + + #[test] + fn nested_values_and_keys_are_cleaned() { + let out = strip_json_nul( + r#"{"o":{"k\u0000":["a\u0000b",{"deep\u0000":"v\u0000"}]}}"#.to_string(), + ); + assert!(!out.contains(r"\u0000")); + let v = parsed(out); + assert_eq!(v["o"]["k"][0], "ab"); + assert_eq!(v["o"]["k"][1]["deep"], "v"); + } + + #[test] + fn odd_backslash_run_keeps_literal_drops_nul() { + // JSON "a\\\u0000b" is an escaped backslash (kept) immediately followed by + // a real NUL escape (dropped) -> decodes to a,backslash,b. + let v = parsed(strip_json_nul(r#"{"x":"a\\\u0000b"}"#.to_string())); + assert_eq!(v["x"], "a\\b"); + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 483541cf2c..c6d978e0d6 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -270,6 +270,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) + .route("/asset_dispatch_edges", get(list_asset_dispatch_edges)) .route( "/list_selected_job_groups", // We use post because sending a huge array as a query param can produce @@ -403,6 +404,7 @@ pub fn workspace_unauthed_service() -> Router { get(get_completed_job_result_maybe), ) .route("/completed/get_timing/{id}", get(get_completed_job_timing)) + .route("/dispatch_events/{id}", get(get_dispatch_events)) .route("/getupdate/{id}", get(get_job_update)) .route("/getupdate_sse/{id}", get(get_job_update_sse)) .route("/get_log_file/{*file_path}", get(get_log_file)) @@ -3202,6 +3204,12 @@ struct ApprovalInfo { #[serde(skip_serializing_if = "Option::is_none")] hide_cancel: Option, approvers: Vec, + /// Share-read-link token for the flow, minted only for callers allowed to view this + /// approval. Lets an authenticated workspace-member approver open the run details of + /// a flow they don't otherwise have read access to (the run page reads it as a + /// `view_token` query param). + #[serde(skip_serializing_if = "Option::is_none")] + view_token: Option, } /// Whether `opt_authed` is allowed to approve — and therefore view — this approval step. @@ -3424,6 +3432,7 @@ async fn get_approval_info( user_auth_required, hide_cancel: None, approvers: vec![], + view_token: None, })); } @@ -3441,6 +3450,12 @@ async fn get_approval_info( }) .collect(); + // Possession of view rights over this approval is sufficient to mint a + // share-read-link token for the flow: it only grants read (no resume), and only to + // an authenticated workspace member, so it never widens what the approver can do. + let hmac = generate_view_token(&w_id, row.id, &db).await?; + let view_token = Some(format!("{}.{hmac}", row.id)); + Ok(Json(ApprovalInfo { flow_id: row.id, form_schema, @@ -3452,6 +3467,7 @@ async fn get_approval_info( user_auth_required, hide_cancel, approvers, + view_token, })) } @@ -3846,6 +3862,12 @@ pub async fn cancel_suspended_job( pub struct SuspendedJobFlow { pub job: Job, pub approvers: Vec, + /// Share-read-link token for the parent flow, minted because the caller proved + /// possession of the approval secret. Lets an authenticated workspace-member + /// approver open the run details of a flow they don't otherwise have read access + /// to (the run page reads it as a `view_token` query param). + #[serde(skip_serializing_if = "Option::is_none")] + pub view_token: Option, } pub async fn get_suspended_job_flow( @@ -3930,7 +3952,13 @@ pub async fn get_suspended_job_flow( ) .await?; - Ok(Json(SuspendedJobFlow { job: flow, approvers }).into_response()) + // Possession of a valid approval secret is sufficient to mint a share-read-link + // token for the parent flow: it only grants read (no resume), and only to an + // authenticated workspace member, so it never widens what the approver can do. + let hmac = generate_view_token(&w_id, flow_id, &db).await?; + let view_token = Some(format!("{flow_id}.{hmac}")); + + Ok(Json(SuspendedJobFlow { job: flow, approvers, view_token }).into_response()) } fn conditionally_require_authed_user( @@ -9032,6 +9060,198 @@ struct JobTiming { duration_ms: Option, } +/// One row of the producer's "Dispatch" panel — what the asset-trigger +/// dispatcher decided for a single (subscriber, asset write) pair. See +/// `windmill_queue::asset_dispatch` for the writer and the discriminants +/// of the `outcome` / `reason` fields. +#[derive(Serialize)] +struct DispatchEvent { + subscriber_path: String, + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + outcome: String, + #[serde(skip_serializing_if = "Option::is_none")] + child_job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + received_inputs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + required_inputs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + debounce_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, + created_at: chrono::DateTime, +} + +async fn get_dispatch_events( + OptViewToken(view_token): OptViewToken, + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult> { + let tags = opt_authed + .as_ref() + .map(|authed| get_scope_tags(authed)) + .flatten(); + + // Gate on the producer job's visibility, exactly like + // get_completed_job_timing on the same unauthed router: scope tags + // first, then per-job read access for authed users, anonymous-only + // jobs otherwise. The dispatch_event FK to v2_job(id) guarantees the + // producer row exists for any extant event. + let producer = sqlx::query!( + r#"SELECT created_by AS "created_by!" + FROM v2_job + WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))"#, + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + ) + .fetch_optional(&db) + .await?; + let producer = not_found_if_none(producer, "Job", id.to_string())?; + + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &producer.created_by, + view_token.as_deref(), + ) + .await?; + } else if producer.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); + } + + let rows = sqlx::query!( + r#"SELECT + subscriber_path AS "subscriber_path!", + asset_kind AS "asset_kind!: windmill_common::assets::AssetKind", + asset_path AS "asset_path!", + outcome::text AS "outcome!", + child_job_id, + partition, + received_inputs, + required_inputs, + debounce_s, + reason, + created_at AS "created_at!" + FROM dispatch_event + WHERE producer_job_id = $1 AND workspace_id = $2 + ORDER BY id"#, + id, + &w_id, + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|r| DispatchEvent { + subscriber_path: r.subscriber_path, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + outcome: r.outcome, + child_job_id: r.child_job_id, + partition: r.partition, + received_inputs: r.received_inputs, + required_inputs: r.required_inputs, + debounce_s: r.debounce_s, + reason: r.reason, + created_at: r.created_at, + }) + .collect(), + )) +} + +/// One asset-cascade dispatch record, for reconstructing the cascade graph of a +/// pipeline folder in the Activity panel. `dispatched` rows carry the resolved +/// `child_job_id` (a real producer→child job edge); `join_pending` rows are the +/// pre-completion inputs of an AND-join (no child yet) — the client links them +/// to the eventual child of the same `subscriber_path` so a join's separate +/// trigger chains merge into one group. `skipped` is omitted. +#[derive(Serialize)] +struct AssetDispatchEdge { + producer_job_id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + child_job_id: Option, + subscriber_path: String, + outcome: String, + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + created_at: chrono::DateTime, +} + +#[derive(Deserialize)] +struct AssetDispatchEdgesQuery { + /// Folder path prefix the children live under, e.g. `f/orders/`. Matched + /// against `subscriber_path` — every intra-pipeline cascade edge has its + /// child in the folder, so this captures the whole folder's cascades. + path_start: String, + /// Only edges dispatched at/after this instant (align with the activity + /// window the client already loaded). Omit for the default cap. + created_after: Option>, +} + +/// Asset-cascade edges for a pipeline folder. RLS on the joined `v2_job` +/// producer row limits this to cascades whose producer the caller can already +/// see (same visibility as the folder's job list). +async fn list_asset_dispatch_edges( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult> { + let like = format!("{}%", query.path_start); + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#"SELECT + de.producer_job_id AS "producer_job_id!", + de.child_job_id, + de.subscriber_path AS "subscriber_path!", + de.outcome::text AS "outcome!", + de.asset_kind AS "asset_kind!: windmill_common::assets::AssetKind", + de.asset_path AS "asset_path!", + de.created_at AS "created_at!" + FROM dispatch_event de + JOIN v2_job pj ON pj.id = de.producer_job_id + WHERE de.workspace_id = $1 + AND de.outcome IN ('dispatched', 'join_pending') + AND de.subscriber_path LIKE $2 + AND ($3::timestamptz IS NULL OR de.created_at >= $3) + ORDER BY de.created_at DESC, de.id DESC + LIMIT 4000"#, + &w_id, + like, + query.created_after, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| AssetDispatchEdge { + producer_job_id: r.producer_job_id, + child_job_id: r.child_job_id, + subscriber_path: r.subscriber_path, + outcome: r.outcome, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + created_at: r.created_at, + }) + .collect(), + )) +} + async fn get_completed_job_timing( OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 2968493ecc..2b70227454 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -1,9 +1,27 @@ -use sqlx::PgExecutor; +use std::collections::HashSet; + +use sqlx::{PgExecutor, Postgres, Transaction}; use crate::{error, scripts::ScriptHash}; +pub use windmill_parser::asset_parser::{parse_pipeline_annotations, TriggerSpec, PARTITION_TOKEN}; pub use windmill_types::assets::*; +#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "SCRIPT_TRIGGER_KIND", rename_all = "lowercase")] +pub enum ScriptTriggerKind { + Asset, + Schedule, + Webhook, + Email, + Kafka, + Mqtt, + Nats, + Postgres, + Sqs, + Gcp, +} + pub async fn insert_static_asset_usage<'e>( executor: impl PgExecutor<'e>, workspace_id: &str, @@ -17,9 +35,22 @@ pub async fn insert_static_asset_usage<'e>( .as_ref() .map(|cols| serde_json::to_value(cols).unwrap_or(serde_json::Value::Null)); + // Invalidate the per-workspace producer-writes cache only when this insert + // actually adds a write producer: the cache (asset_dispatch:: + // ASSET_PRODUCER_WRITES_CACHE) tracks script rows with 'w'/'rw' access, so + // a row that was a no-op (ON CONFLICT skipped), a flow usage, or read-only + // can't change it. Emitting in the same statement keeps the notify atomic + // with the insert and visible to pollers only on commit. See the matching + // delete-side guard in clear_static_asset_usage. sqlx::query!( - r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) - VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING"#, + r#"WITH ins AS ( + INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING + RETURNING usage_kind, usage_access_type + ) + INSERT INTO notify_event (channel, payload) + SELECT 'notify_asset_producer_change', $1 + FROM ins WHERE usage_kind = 'script' AND usage_access_type IN ('w', 'rw')"#, workspace_id, asset.path, asset.kind as AssetKind, @@ -40,8 +71,25 @@ pub async fn clear_static_asset_usage<'e>( usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { + // Invalidate the per-workspace producer-writes cache that gates the + // asset-trigger dispatch hook (windmill-queue + // asset_dispatch::ASSET_PRODUCER_WRITES_CACHE). That cache only tracks + // script rows with 'w'/'rw' access, so emit the notify only when the + // delete actually removed such a write producer: flow usage, read-only + // usage, and deletes that matched no producer row leave the cache + // unchanged (the common case — most deploys touch no write asset). The + // matching add side lives in insert_static_asset_usage. Emitting in the + // same statement keeps the notify atomic with the delete and visible to + // pollers only on commit. sqlx::query!( - r#"DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3"#, + r#"WITH del AS ( + DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3 + RETURNING usage_access_type + ) + INSERT INTO notify_event (channel, payload) + SELECT 'notify_asset_producer_change', $1 + WHERE $3 = 'script' + AND EXISTS (SELECT 1 FROM del WHERE usage_access_type IN ('w', 'rw'))"#, workspace_id, usage_path, usage_kind as AssetUsageKind @@ -56,8 +104,18 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( workspace_id: &str, script_hash: ScriptHash, ) -> error::Result<()> { + // Always script usage → invalidate the producer-writes cache for this + // workspace, but only when the delete actually removed a write producer + // ('w'/'rw'); see clear_static_asset_usage. Atomic with the delete. sqlx::query!( - "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + r#"WITH del AS ( + DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' + AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1) + RETURNING usage_access_type + ) + INSERT INTO notify_event (channel, payload) + SELECT 'notify_asset_producer_change', $1 + WHERE EXISTS (SELECT 1 FROM del WHERE usage_access_type IN ('w', 'rw'))"#, workspace_id, script_hash.0 ) @@ -66,6 +124,253 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( Ok(()) } +fn is_write_access(access: Option) -> bool { + matches!( + access, + Some(AssetUsageAccessType::W) | Some(AssetUsageAccessType::RW) + ) +} + +/// Clear and reinsert the full static-asset usage set of a script in one tx, +/// invalidating the producer-writes cache at most once and only on a real +/// change. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) keys a +/// workspace by the set of (kind, path) script rows with 'w'/'rw' access, so a +/// redeploy that keeps the same write producers must leave it untouched. We +/// diff the old write set (captured from the clearing DELETE's RETURNING) +/// against the new one and emit a single notify only when they differ — +/// emitting per statement, as clear/insert_static_asset_usage do, would fire +/// twice on every write-asset redeploy (the clear removes the row, the reinsert +/// adds it back). The notify rides the deploy tx, so it stays atomic with the +/// DML and visible to pollers only on commit. +/// +/// DELETE and INSERT of the same primary key cannot share a single statement +/// (both would read the pre-statement snapshot, so the reinsert's ON CONFLICT +/// would silently drop the row), which is why this clears and reinserts as +/// separate statements rather than one CTE. +/// +/// Performs no authorization itself: `tx` must already be scoped to a caller +/// authorized for `workspace_id`/`usage_path` (e.g. `user_db.begin(&authed)`, +/// which applies RLS), exactly like the sibling clear/insert helpers. +pub async fn replace_static_asset_usage( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &str, + usage_path: &str, + assets: &[AssetWithAltAccessType], +) -> error::Result<()> { + let cleared = sqlx::query!( + r#"DELETE FROM asset + WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = 'script' + RETURNING kind AS "kind!: AssetKind", path, + usage_access_type AS "usage_access_type: AssetUsageAccessType""#, + workspace_id, + usage_path, + ) + .fetch_all(&mut **tx) + .await?; + let old_writes: HashSet<(AssetKind, String)> = cleared + .into_iter() + .filter(|r| is_write_access(r.usage_access_type)) + .map(|r| (r.kind, r.path)) + .collect(); + + // Build new_writes from rows actually inserted (RETURNING), not the + // requested slice: ON CONFLICT DO NOTHING is first-writer-wins, so a payload + // with duplicate (kind, path) entries at conflicting access types persists + // only the first. Since the DELETE above emptied this usage_path, every + // non-conflicting insert lands, so the inserted rows are exactly the new + // persisted set. + let mut new_writes: HashSet<(AssetKind, String)> = HashSet::new(); + for asset in assets { + let access = asset.access_type.or(asset.alt_access_type); + let columns_json = asset + .columns + .as_ref() + .map(|cols| serde_json::to_value(cols).unwrap_or(serde_json::Value::Null)); + let inserted = sqlx::query!( + r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) + VALUES ($1, $2, $3, $4, $5, 'script', $6) ON CONFLICT DO NOTHING + RETURNING usage_access_type AS "usage_access_type: AssetUsageAccessType""#, + workspace_id, + asset.path, + asset.kind as AssetKind, + access as Option, + usage_path, + columns_json as Option, + ) + .fetch_optional(&mut **tx) + .await?; + if let Some(row) = inserted { + if is_write_access(row.usage_access_type) { + new_writes.insert((asset.kind, asset.path.clone())); + } + } + } + + if old_writes != new_writes { + sqlx::query!( + r#"INSERT INTO notify_event (channel, payload) + VALUES ('notify_asset_producer_change', $1)"#, + workspace_id, + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +// Wipe all pipeline trigger declarations held by the given runnable. Used at +// deploy time: redeploying a script wipes its prior `// on` annotations so +// removing them implicitly un-declares those edges. +pub async fn clear_script_triggers<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + runnable_path: &str, + runnable_kind: AssetUsageKind, +) -> error::Result<()> { + sqlx::query!( + r#"DELETE FROM script_trigger + WHERE workspace_id = $1 AND runnable_kind = $2 AND runnable_path = $3"#, + workspace_id, + runnable_kind as AssetUsageKind, + runnable_path, + ) + .execute(executor) + .await?; + Ok(()) +} + +// Insert a single trigger declaration. Caller is expected to wipe first. +// `join_all` is the script-level `// trigger all` flag (AND join barrier); +// `retry_count` / `retry_delay_s` are the `// retry []` policy. +// All three are script-level — the same value for every row of a given +// runnable — but stored per-row to keep the wipe-and-reinsert pattern and a +// single-query subscriber lookup. +pub async fn insert_script_trigger<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + runnable_kind: AssetUsageKind, + runnable_path: &str, + trigger_kind: ScriptTriggerKind, + trigger_ref: &str, + join_all: bool, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +) -> error::Result<()> { + sqlx::query!( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, + debounce_s, retry_count, retry_delay_s) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#, + workspace_id, + runnable_kind as AssetUsageKind, + runnable_path, + trigger_kind as ScriptTriggerKind, + trigger_ref, + join_all, + debounce_s, + retry_count, + retry_delay_s, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// Parse a debounce duration into whole seconds. Accepts a bare integer +/// (seconds) or an `` with an `s`/`m`/`h`/`d` suffix (e.g. `30s`, +/// `5m`, `2h`, `1d`). Returns `None` for empty / malformed / non-positive +/// input — the caller treats `None` as "no debounce" (fan-out), so a typo +/// fails safe rather than silently debouncing. +pub fn parse_duration_secs(s: &str) -> Option { + let s = s.trim(); + if s.is_empty() { + return None; + } + let (num, mult): (&str, i64) = match s.as_bytes().last() { + Some(b's') => (&s[..s.len() - 1], 1), + Some(b'm') => (&s[..s.len() - 1], 60), + Some(b'h') => (&s[..s.len() - 1], 3600), + Some(b'd') => (&s[..s.len() - 1], 86400), + Some(c) if c.is_ascii_digit() => (s, 1), + _ => return None, + }; + let n: i64 = num.trim().parse().ok()?; + let secs = n.checked_mul(mult)?; + if secs <= 0 || secs > i32::MAX as i64 { + return None; + } + Some(secs as i32) +} + +#[cfg(test)] +mod debounce_duration_tests { + use super::parse_duration_secs; + + #[test] + fn parses_units_and_bare_seconds() { + assert_eq!(parse_duration_secs("60"), Some(60)); + assert_eq!(parse_duration_secs("30s"), Some(30)); + assert_eq!(parse_duration_secs("5m"), Some(300)); + assert_eq!(parse_duration_secs("2h"), Some(7200)); + assert_eq!(parse_duration_secs(" 1d "), Some(86400)); + } + + #[test] + fn rejects_garbage_and_nonpositive() { + assert_eq!(parse_duration_secs(""), None); + assert_eq!(parse_duration_secs("abc"), None); + assert_eq!(parse_duration_secs("0"), None); + assert_eq!(parse_duration_secs("-5"), None); + assert_eq!(parse_duration_secs("10x"), None); + assert_eq!(parse_duration_secs("s"), None); + } +} + +// Inverse of trigger_spec_to_row for the Asset variant: parses a stored +// trigger_ref (e.g. `s3://foo`, `$res:bar`) back into the (kind, path) pair +// used as a graph node id. Returns None for refs that don't match any known +// asset prefix — callers should skip those edges. +pub fn parse_asset_trigger_ref(s: &str) -> Option<(AssetKind, String)> { + let (kind, path) = windmill_parser::asset_parser::parse_asset_syntax(s, false)?; + Some((asset_kind_from_parser(kind), path.to_string())) +} + +// Convert a parser TriggerSpec into the `(kind, ref)` pair stored in +// script_trigger. Asset refs get their canonical prefix back so the +// trigger_ref matches what downstream lookups expect. +// +// Returns `None` for native trigger kinds (Kafka, Mqtt, Postgres, …) — +// those annotations are marker-only and don't produce a `script_trigger` +// row. The actual binding lives on the trigger row's own `script_path` +// column; the graph endpoint looks it up directly per kind. +pub fn trigger_spec_to_row(spec: &TriggerSpec) -> Option<(ScriptTriggerKind, String)> { + match spec { + TriggerSpec::Asset { asset_kind, path, .. } => { + // Single source of truth for the canonical prefix lives on the + // common AssetKind; map the parser kind across first. The parser + // enum has no Variable variant, so canonical_prefix is always Some. + let prefix = asset_kind_from_parser(*asset_kind).canonical_prefix()?; + Some((ScriptTriggerKind::Asset, format!("{}{}", prefix, path))) + } + // Schedule joins the native-trigger family — no script_trigger row + // is inserted for the annotation. The binding lives on the schedule + // row's own `script_path` field, same as kafka/mqtt/etc. + TriggerSpec::Schedule + | TriggerSpec::Webhook + | TriggerSpec::Email + | TriggerSpec::Kafka + | TriggerSpec::Mqtt + | TriggerSpec::Nats + | TriggerSpec::Postgres + | TriggerSpec::Sqs + | TriggerSpec::Gcp + // data_upload is a UI-first marker — no event source, no trigger row. + // The script's S3Object input + auto-generated S3 picker drive it. + | TriggerSpec::DataUpload => None, + } +} + pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetKind) -> AssetKind { match parser_kind { windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 732b9cdc30..4d1d48a53d 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -1,3 +1,8 @@ +// Adding a global setting? Decide whether agent workers may read it. Agent +// workers (remote workers connected over HTTP) fetch settings through an +// endpoint that is deny-by-exception: every key is served except those in +// AGENT_WORKER_BLOCKED_SETTINGS (defined below). If a new setting holds an +// instance secret the server should keep to itself, add its key there. pub const CUSTOM_TAGS_SETTING: &str = "custom_tags"; pub const DEFAULT_TAGS_PER_WORKSPACE_SETTING: &str = "default_tags_per_workspace"; pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; @@ -71,6 +76,7 @@ pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; +pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; @@ -104,6 +110,53 @@ pub const WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING: &str = "workspace_fairness_max pub const WORKSPACE_FAIRNESS_DURATION_SECS_SETTING: &str = "workspace_fairness_duration_secs"; pub const WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING: &str = "workspace_fairness_min_total_jobs"; +/// Global settings an agent worker (a remote worker connected over HTTP instead +/// of to the database) must NEVER read through +/// `GET /api/agent_workers/get_global_setting/{key}`. Every other key is served. +/// +/// SECURITY: that endpoint is authenticated only by an agent-worker JWT and +/// returns the raw `global_settings` value for the requested key. Because the +/// policy is deny-by-exception (anything not listed here is readable), every +/// setting that holds an instance secret or credential an agent worker does not +/// need MUST be listed below. Missing one discloses it to every agent worker — +/// `jwt_secret` is the worst case (a token holder could forge a superadmin JWT), +/// but `oauths`, `smtp_settings`, `secret_backend`, object-store credentials, +/// etc. are instance-wide secrets too. +/// +/// NOT blocked, on purpose: the operational credentials an agent worker loads to +/// run jobs (`license_key`, `hub_api_secret`, `sandbox_registry_auth`, +/// `powershell_repo_pat`, `npmrc`, ...). Those are already within an agent +/// worker's trust boundary, and blocking them breaks worker startup or +/// dependency installation. When adding a new setting that stores a secret the +/// server keeps to itself, add it here. +pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ + // Instance identity / auth secrets — disclosure enables privilege escalation + // or impersonation. + JWT_SECRET_SETTING, + OAUTH_SETTING, + SMTP_SETTING, + SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, + SECRET_BACKEND_SETTING, + GITHUB_ENTERPRISE_APP_SETTING, + OBJECT_STORE_CONFIG_SETTING, + AI_CONFIG_SETTING, + TEAMS_SETTING, + INDEXER_SETTING, + // Server-only configs that may embed credentials, webhook URLs or tokens and + // are never loaded by an agent worker. + CRITICAL_ERROR_CHANNELS_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, + OTEL_SETTING, + OTEL_TRACING_PROXY_SETTING, +]; + +/// Whether an agent worker may read the given global setting over HTTP. +/// Deny-by-exception: everything is readable except [`AGENT_WORKER_BLOCKED_SETTINGS`]. +pub fn is_setting_readable_by_agent_worker(name: &str) -> bool { + !AGENT_WORKER_BLOCKED_SETTINGS.contains(&name) +} + use std::sync::atomic::AtomicBool; lazy_static::lazy_static! { @@ -265,6 +318,62 @@ pub fn workspace_integration_auth_endpoint(client_name: &str, base_url: &str) -> } } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn agent_workers_can_read_operational_settings() { + // Operational knobs and the credentials a worker needs to run jobs are + // intentionally NOT blocked. Deny-by-exception also means an arbitrary + // unlisted key is readable. + for key in [ + NPMRC_SETTING, + PIP_INDEX_URL_SETTING, + JOB_ISOLATION_SETTING, + LICENSE_KEY_SETTING, + HUB_API_SECRET_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, + POWERSHELL_REPO_PAT_SETTING, + "some_future_operational_setting", + ] { + assert!( + is_setting_readable_by_agent_worker(key), + "'{key}' must remain readable by agent workers" + ); + } + } + + #[test] + fn agent_workers_cannot_read_instance_secrets() { + // Disclosing any of these to a remote worker enables privilege + // escalation (jwt_secret -> forged superadmin JWT) or leaks instance + // secrets. They must never be served by the agent-worker endpoint. + for key in [ + JWT_SECRET_SETTING, + OAUTH_SETTING, + SMTP_SETTING, + SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, + SECRET_BACKEND_SETTING, + GITHUB_ENTERPRISE_APP_SETTING, + OBJECT_STORE_CONFIG_SETTING, + AI_CONFIG_SETTING, + TEAMS_SETTING, + INDEXER_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, + OTEL_SETTING, + OTEL_TRACING_PROXY_SETTING, + ] { + assert!( + !is_setting_readable_by_agent_worker(key), + "'{key}' is an instance secret and must not be readable by agent workers" + ); + } + } +} + pub async fn set_value_in_global_settings( db: &Pool, setting_name: &str, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 1f7611a13a..7c2a475361 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -61,6 +61,7 @@ pub mod indexer; pub mod instance_config; pub mod job_metrics; pub mod log_context; +pub mod materialization; pub mod min_version; pub mod notify_events; pub mod runtime_assets; @@ -81,6 +82,20 @@ pub mod oidc_oss; #[cfg(feature = "private")] pub mod otel_ee; pub mod otel_oss; +#[cfg(feature = "private")] +pub mod partition_ee; +pub mod partition_oss; +#[cfg(feature = "private")] +pub use partition_ee as partition; +#[cfg(not(feature = "private"))] +pub use partition_oss as partition; +#[cfg(feature = "private")] +pub mod pipeline_advanced_ee; +pub mod pipeline_advanced_oss; +#[cfg(feature = "private")] +pub use pipeline_advanced_ee as pipeline_advanced; +#[cfg(not(feature = "private"))] +pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; pub mod result_stream; diff --git a/backend/windmill-common/src/materialization.rs b/backend/windmill-common/src/materialization.rs new file mode 100644 index 0000000000..ccc1f7782e --- /dev/null +++ b/backend/windmill-common/src/materialization.rs @@ -0,0 +1,135 @@ +//! CE materialization state — the per-partition status recorded by the managed +//! `// materialize` write (in windmill-worker), read by the partition-status +//! grid and by the EE backfill worklist. +//! +//! The write engine and this state are CE; only automatic partition +//! *resolution* (`partition_ee`) and *backfill* orchestration +//! (`pipeline_advanced_ee`) are enterprise. This module is the shared seam: +//! the EE backfill enumerates the partitions in a range, diffs them against +//! these rows to find the missing/failed set, and pushes one CE materialization +//! job per gap (with an explicit `partition` arg — which runs idempotently and +//! upserts the row here). Nothing about that orchestration lives in this file; +//! it only needs the rows to exist, which is why recording is CE. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::PgExecutor; +use uuid::Uuid; + +use crate::assets::AssetKind; +use crate::error::Result; + +/// Sentinel `partition` value for an unpartitioned (whole-table) +/// materialization — partition is part of the primary key and cannot be NULL. +pub const UNPARTITIONED: &str = ""; + +/// Mirrors the `MATERIALIZATION_STATUS` pg enum (see migration +/// `20260619170118_add_materialized_partition`). +#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[sqlx(type_name = "MATERIALIZATION_STATUS", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MaterializationStatus { + Running, + Materialized, + Failed, +} + +/// The materialization outcome an agent worker (`Connection::Http`, no direct +/// DB) sends to the API to be recorded. Mirrors the `record_materialization` +/// args; the API handler unpacks it and calls that function with its own DB. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecordMaterializationRequest { + pub asset_kind: AssetKind, + pub asset_path: String, + pub partition: String, + pub status: MaterializationStatus, + pub snapshot_id: Option, + pub row_count: Option, + pub job_id: Option, + pub error: Option, +} + +/// Upsert the latest materialization state for one (asset, partition) slice. +/// The worker records the terminal outcome once the write finishes: +/// `Materialized` (with the DuckLake `snapshot_id` + `row_count`) or `Failed` +/// (with `error`). `Running` mirrors the pg enum but has no writer in this flow. +/// Idempotent: re-running the same partition overwrites the row — exactly the +/// backfill / failure-recovery contract. +#[allow(clippy::too_many_arguments)] +pub async fn record_materialization<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_kind: AssetKind, + asset_path: &str, + partition: &str, + status: MaterializationStatus, + snapshot_id: Option, + row_count: Option, + job_id: Option, + error: Option<&str>, +) -> Result<()> { + sqlx::query!( + "INSERT INTO materialized_partition + (workspace_id, asset_kind, asset_path, partition, status, + snapshot_id, row_count, job_id, materialized_at, error) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9) + ON CONFLICT (workspace_id, asset_kind, asset_path, partition) + DO UPDATE SET status = EXCLUDED.status, + snapshot_id = EXCLUDED.snapshot_id, + row_count = EXCLUDED.row_count, + job_id = EXCLUDED.job_id, + materialized_at = now(), + error = EXCLUDED.error", + workspace_id, + asset_kind as AssetKind, + asset_path, + partition, + status as MaterializationStatus, + snapshot_id, + row_count, + job_id, + error, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// One materialized-partition row, for the status grid / backfill diff. +#[derive(sqlx::FromRow, Debug, Clone, Serialize)] +pub struct MaterializedPartition { + pub asset_kind: AssetKind, + pub asset_path: String, + pub partition: String, + pub status: MaterializationStatus, + pub snapshot_id: Option, + pub row_count: Option, + pub job_id: Option, + pub materialized_at: DateTime, + pub error: Option, +} + +/// All recorded partitions for one asset, newest first — the grid's data and +/// the backfill worklist's "what already exists" set. +pub async fn list_materialized_partitions<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + asset_kind: AssetKind, + asset_path: &str, +) -> Result> { + let rows = sqlx::query_as!( + MaterializedPartition, + r#"SELECT asset_kind AS "asset_kind: AssetKind", asset_path, partition, + status AS "status: MaterializationStatus", snapshot_id, + row_count, job_id, materialized_at, error + FROM materialized_partition + WHERE workspace_id = $1 AND asset_kind = $2 AND asset_path = $3 + ORDER BY partition DESC"#, + workspace_id, + asset_kind as AssetKind, + asset_path, + ) + .fetch_all(executor) + .await?; + Ok(rows) +} diff --git a/backend/windmill-common/src/partition_oss.rs b/backend/windmill-common/src/partition_oss.rs new file mode 100644 index 0000000000..0e9c05ac36 --- /dev/null +++ b/backend/windmill-common/src/partition_oss.rs @@ -0,0 +1,55 @@ +//! OSS fallback for pipeline partition resolution. +//! +//! Partition resolution is a `private` feature (see `partition_ee`). In the +//! public build it is absent: `resolve_partition` yields no partition, so the +//! cascade runs partition-agnostically, and the args helpers degrade to a +//! plain (non-partition-preserving) write. The real implementation lives in +//! `windmill-ee-private`. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::PgExecutor; +use uuid::Uuid; +use windmill_parser::asset_parser::PartitionSpec; + +use crate::error::Result; + +/// Well-known arg key the resolved partition value is injected under. +pub const PARTITION_ARG: &str = "partition"; + +/// No-op: OSS never resolves a partition, so there is nothing to persist. +pub async fn set_resolved_partition<'e>( + _executor: impl PgExecutor<'e>, + _job_id: Uuid, + _value: &str, +) -> Result<()> { + Ok(()) +} + +/// Plain args replace — OSS has no resolved partition to carry forward. +pub async fn merge_args_preserving_partition<'e>( + executor: impl PgExecutor<'e>, + job_id: Uuid, + new_args: HashMap>, +) -> Result<()> { + sqlx::query!( + "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + Json(new_args) as Json>>, + job_id, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// No partition in the OSS build. +pub fn resolve_partition( + _spec: &PartitionSpec, + _at: DateTime, + _payload: Option<&serde_json::Value>, +) -> Result> { + Ok(None) +} diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs new file mode 100644 index 0000000000..009e783bd5 --- /dev/null +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -0,0 +1,16 @@ +//! OSS fallback: pipeline freshness/SLA enforcement and partition backfills +//! are enterprise features; their implementations live in windmill-ee-private +//! (see `pipeline_advanced_ee`). In the public build the entry points report +//! that the enterprise edition is required. + +use crate::error::Error; + +pub fn freshness_enforcement_todo() -> Error { + Error::internal_err( + "Pipeline freshness/SLA enforcement requires the enterprise edition".to_string(), + ) +} + +pub fn backfill_todo() -> Error { + Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) +} diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index f077687d6f..2c168cd433 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -1739,7 +1739,7 @@ struct PrimaryKeyConstraintPayload { fn db_supports_schemas(db_type: DbType) -> bool { matches!( db_type, - DbType::Postgresql | DbType::Snowflake | DbType::Bigquery + DbType::Postgresql | DbType::Snowflake | DbType::Bigquery | DbType::Duckdb ) } @@ -2410,8 +2410,15 @@ fn make_load_table_metadata_query( ) -> Result { match db_type { DbType::Duckdb => { - // For ducklake, the ducklake ATTACH is handled by the ducklake wrapper. - let mut q = String::from( + // For ducklake, the ducklake ATTACH is handled by the ducklake wrapper, so the + // ducklake catalog is the current database. information_schema spans every attached + // catalog, so we always scope to current_database() to stay within the ducklake. + let extra_col = if table.is_none() { + ",\n TABLE_SCHEMA as schema_name" + } else { + "" + }; + let mut q = format!( "SELECT COLUMN_NAME as field, DATA_TYPE as DataType, @@ -2420,12 +2427,20 @@ fn make_load_table_metadata_query( false as IsIdentity, CASE WHEN IS_NULLABLE = true THEN 'YES' ELSE 'NO' END as IsNullable, false as IsEnum, - TABLE_NAME as table_name + TABLE_NAME as table_name{} FROM information_schema.columns c -WHERE table_schema = current_schema()", +WHERE table_catalog = current_database()", + extra_col ); if let Some(t) = table { - q.push_str(&format!(" AND TABLE_NAME = '{}'", escape_sql_literal(t))); + let parts: Vec<&str> = t.split('.').collect(); + let tname = parts[parts.len() - 1]; + let schema = if parts.len() > 1 { parts[0] } else { "main" }; + q.push_str(&format!( + " AND TABLE_NAME = '{}' AND TABLE_SCHEMA = '{}'", + escape_sql_literal(tname), + escape_sql_literal(schema) + )); } Ok(q) } @@ -3722,9 +3737,10 @@ mod tests { table_ref("users", Some("myschema"), DbType::Mysql), "`users`" ); + // DuckDB (ducklake) supports schemas assert_eq!( table_ref("users", Some("myschema"), DbType::Duckdb), - r#""users""# + r#""myschema"."users""# ); } @@ -3856,6 +3872,13 @@ mod tests { assert!(sql.contains("DROP TABLE \"users\";")); } + #[test] + fn test_expand_drop_table_ducklake_with_schema() { + let marker = r#"-- WM_INTERNAL_DB_DROP_TABLE {"table":"events","schema":"analytics","ducklake":"my_lake"}"#; + let sql = expand_code(marker, &ScriptLang::DuckDb); + assert!(sql.contains("DROP TABLE \"analytics\".\"events\";")); + } + // ----------------------------------------------------------------------- // CREATE SCHEMA / DROP SCHEMA // ----------------------------------------------------------------------- @@ -4355,7 +4378,27 @@ mod tests { let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"table":"users","ducklake":"lake"}"#; let sql = expand_code(marker, &ScriptLang::DuckDb); assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n")); - assert!(sql.contains("TABLE_NAME = 'users'")); + assert!(sql.contains("table_catalog = current_database()")); + // Unqualified table defaults to the "main" schema. + assert!(sql.contains("TABLE_NAME = 'users' AND TABLE_SCHEMA = 'main'")); + } + + #[test] + fn test_expand_load_table_metadata_ducklake_qualified_schema() { + let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"table":"analytics.events","ducklake":"lake"}"#; + let sql = expand_code(marker, &ScriptLang::DuckDb); + assert!(sql.contains("TABLE_NAME = 'events' AND TABLE_SCHEMA = 'analytics'")); + } + + #[test] + fn test_expand_load_table_metadata_ducklake_all_tables() { + let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"ducklake":"lake"}"#; + let sql = expand_code(marker, &ScriptLang::DuckDb); + assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n")); + // All-tables listing scopes to the ducklake catalog and exposes the schema per table. + assert!(sql.contains("table_catalog = current_database()")); + assert!(sql.contains("TABLE_SCHEMA as schema_name")); + assert!(!sql.contains("TABLE_NAME = '")); } // ----------------------------------------------------------------------- diff --git a/backend/windmill-common/src/runtime_assets.rs b/backend/windmill-common/src/runtime_assets.rs index ecd863f482..70185ca48c 100644 --- a/backend/windmill-common/src/runtime_assets.rs +++ b/backend/windmill-common/src/runtime_assets.rs @@ -127,12 +127,19 @@ async fn prune_runtime_assets( .map(|((w, p, k), v)| (w.clone(), p.clone(), k.clone(), (max_n - v.len()) as i32)) .multiunzip(); + // Delete the surplus job rows by `id`, NOT by `(workspace_id, path, + // kind)`. Deleting by tuple would also wipe the static + // `usage_kind='script'|'flow'` rows that share the same path+kind (a + // producer's persisted write/read lineage), which silently breaks the + // asset-trigger cascade (`fetch_producer_writes` finds no writes). The + // inner query already scopes to `usage_kind='job'`; keep the delete + // scoped to exactly those over-cap rows. let delete_result = sqlx::query!( r#" DELETE FROM asset - WHERE (workspace_id, path, kind) IN ( - SELECT workspace_id, path, kind FROM ( - SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER ( + WHERE id IN ( + SELECT id FROM ( + SELECT a.id, ROW_NUMBER() OVER ( PARTITION BY a.workspace_id, a.path, a.kind ORDER BY a.created_at DESC ) as rn, @@ -140,14 +147,14 @@ async fn prune_runtime_assets( FROM asset a INNER JOIN ( SELECT * FROM UNNEST( - $1::varchar[], - $2::varchar[], + $1::varchar[], + $2::varchar[], $3::asset_kind[], $4::int[] ) AS t(workspace_id, path, kind, max_n) ) limits - ON a.workspace_id = limits.workspace_id - AND a.path = limits.path + ON a.workspace_id = limits.workspace_id + AND a.path = limits.path AND a.kind = limits.kind WHERE a.usage_kind = 'job' ) ranked diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index b99ede353c..620cbe2a5f 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -53,6 +53,10 @@ pub enum UserDraftItemKind { TriggerNextcloud, TriggerGoogle, TriggerGithub, + /// All unsaved scripts of one data pipeline, bundled into a single draft + /// keyed at the pipeline's folder path. Not a runnable: it has no deployed + /// backing table and is private to its owner. + DataPipeline, } impl UserDraftItemKind { @@ -84,12 +88,13 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerNextcloud => "trigger_nextcloud", UserDraftItemKind::TriggerGoogle => "trigger_google", UserDraftItemKind::TriggerGithub => "trigger_github", + UserDraftItemKind::DataPipeline => "data_pipeline", } } /// Every variant, for code that must enumerate kinds (e.g. generating /// the `draft_only` existence SQL). - pub const ALL: [UserDraftItemKind; 24] = [ + pub const ALL: [UserDraftItemKind; 25] = [ UserDraftItemKind::Script, UserDraftItemKind::Flow, UserDraftItemKind::App, @@ -114,6 +119,7 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerNextcloud, UserDraftItemKind::TriggerGoogle, UserDraftItemKind::TriggerGithub, + UserDraftItemKind::DataPipeline, ]; /// The deployed table backing this kind, keyed by `(workspace_id, path)`. @@ -144,6 +150,9 @@ impl UserDraftItemKind { TriggerEmail | TriggerDefaultEmail => Some("email_trigger"), TriggerWebhook | TriggerPoll | TriggerCli | TriggerNextcloud | TriggerGoogle | TriggerGithub => None, + // Keyed at a folder path, not a runnable; access falls back to the + // path-only (folder write) check. + DataPipeline => None, } } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 85fd7237a9..6a7ecb40a6 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -166,7 +166,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28261/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28719/sync-script-to-git-repo-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/backend/windmill-common/tests/asset_producer_notify.rs b/backend/windmill-common/tests/asset_producer_notify.rs new file mode 100644 index 0000000000..d09b0114eb --- /dev/null +++ b/backend/windmill-common/tests/asset_producer_notify.rs @@ -0,0 +1,474 @@ +/*! + * Tests that the asset producer-writes cache invalidation + * (`notify_asset_producer_change`) is emitted only when a deploy actually + * changes the set of script write-producers ('w'/'rw' asset usage), not on + * every deploy. The cache (asset_dispatch::ASSET_PRODUCER_WRITES_CACHE) only + * tracks script rows with write access, so read-only usage, flow usage, and + * deploys touching no write asset must NOT emit an event. + */ + +use sqlx::{Pool, Postgres}; +use windmill_common::assets::{ + clear_static_asset_usage, clear_static_asset_usage_by_script_hash, insert_static_asset_usage, + replace_static_asset_usage, AssetKind, AssetUsageAccessType, AssetUsageKind, + AssetWithAltAccessType, +}; +use windmill_common::scripts::ScriptHash; + +const WS: &str = "test-workspace"; + +/// Run the deploy-time clear+reinsert against `usage_path` in its own tx, +/// mirroring how create_script_internal calls it. +async fn replace(db: &Pool, usage_path: &str, assets: &[AssetWithAltAccessType]) { + let mut tx = db.begin().await.expect("begin"); + replace_static_asset_usage(&mut tx, WS, usage_path, assets) + .await + .expect("replace static asset usage"); + tx.commit().await.expect("commit"); +} + +fn asset_at( + path: &str, + kind: AssetKind, + access: Option, +) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind, + access_type: access, + alt_access_type: None, + columns: None, + } +} + +async fn producer_notify_count(db: &Pool) -> i64 { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM notify_event WHERE channel = 'notify_asset_producer_change'", + ) + .fetch_one(db) + .await + .expect("count notify events") +} + +async fn reset_notify(db: &Pool) { + sqlx::query("DELETE FROM notify_event WHERE channel = 'notify_asset_producer_change'") + .execute(db) + .await + .expect("reset notify events"); +} + +fn asset(access: Option) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: "u/test-user/res".to_string(), + kind: AssetKind::Resource, + access_type: access, + alt_access_type: None, + columns: None, + } +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn insert_write_script_asset_emits(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::W)), + "u/test-user/script", + AssetUsageKind::Script, + ) + .await + .unwrap(); + assert_eq!(producer_notify_count(&db).await, 1, "write asset must emit"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn insert_readonly_script_asset_does_not_emit(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::R)), + "u/test-user/script", + AssetUsageKind::Script, + ) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "read-only asset is not a write producer" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn insert_write_flow_asset_does_not_emit(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::W)), + "u/test-user/flow", + AssetUsageKind::Flow, + ) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "flow usage does not affect the script producer cache" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn clear_plain_script_does_not_emit(db: Pool) { + // No asset rows for this path: a plain script deploy must not emit. + clear_static_asset_usage(&db, WS, "u/test-user/plain", AssetUsageKind::Script) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "plain script deploy must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn clear_removes_write_producer_emits(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::RW)), + "u/test-user/script", + AssetUsageKind::Script, + ) + .await + .unwrap(); + reset_notify(&db).await; // ignore the insert-side event; isolate the clear + + clear_static_asset_usage(&db, WS, "u/test-user/script", AssetUsageKind::Script) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 1, + "removing a write producer must emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn clear_removes_only_readonly_does_not_emit(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::R)), + "u/test-user/script", + AssetUsageKind::Script, + ) + .await + .unwrap(); + reset_notify(&db).await; + + clear_static_asset_usage(&db, WS, "u/test-user/script", AssetUsageKind::Script) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "removing only read-only usage must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn clear_flow_usage_does_not_emit(db: Pool) { + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::W)), + "u/test-user/flow", + AssetUsageKind::Flow, + ) + .await + .unwrap(); + reset_notify(&db).await; + + clear_static_asset_usage(&db, WS, "u/test-user/flow", AssetUsageKind::Flow) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "clearing flow usage must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn clear_by_script_hash_gated_on_write(db: Pool) { + let hash = 123456789_i64; + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, lock) + VALUES ($1, $2, 'u/test-user/byhash', '', '', '', 'test-user', 'deno', '')"#, + ) + .bind(WS) + .bind(hash) + .execute(&db) + .await + .unwrap(); + + // Read-only usage on that script path → clear must not emit. + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::R)), + "u/test-user/byhash", + AssetUsageKind::Script, + ) + .await + .unwrap(); + reset_notify(&db).await; + + clear_static_asset_usage_by_script_hash(&db, WS, ScriptHash(hash)) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 0, + "by-hash clear of read-only usage must not emit" + ); + + // Now a write usage → by-hash clear must emit. + insert_static_asset_usage( + &db, + WS, + &asset(Some(AssetUsageAccessType::W)), + "u/test-user/byhash", + AssetUsageKind::Script, + ) + .await + .unwrap(); + reset_notify(&db).await; + + clear_static_asset_usage_by_script_hash(&db, WS, ScriptHash(hash)) + .await + .unwrap(); + assert_eq!( + producer_notify_count(&db).await, + 1, + "by-hash clear of write usage must emit" + ); +} + +const SP: &str = "u/test-user/script"; + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_plain_deploy_does_not_emit(db: Pool) { + replace(&db, SP, &[]).await; + assert_eq!( + producer_notify_count(&db).await, + 0, + "deploy with no assets must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_gaining_write_emits_once(db: Pool) { + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + )], + ) + .await; + assert_eq!( + producer_notify_count(&db).await, + 1, + "gaining a write producer must emit exactly once" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_unchanged_write_set_does_not_emit(db: Pool) { + let assets = [asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + )]; + replace(&db, SP, &assets).await; + reset_notify(&db).await; // isolate the redeploy + + // Redeploy with the identical write-producer set: clear removes the row and + // the reinsert adds it back, but the cache value is unchanged → no emit. + replace(&db, SP, &assets).await; + assert_eq!( + producer_notify_count(&db).await, + 0, + "redeploy keeping the same write producers must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_dropping_write_emits(db: Pool) { + let assets = [asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::RW), + )]; + replace(&db, SP, &assets).await; + reset_notify(&db).await; + + // Redeploy with no assets: the write producer disappears → emit. + replace(&db, SP, &[]).await; + assert_eq!( + producer_notify_count(&db).await, + 1, + "dropping the last write producer must emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_readonly_to_write_emits(db: Pool) { + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::R), + )], + ) + .await; + reset_notify(&db).await; + + // Same asset path, access flips read-only → write: cache gains a row → emit. + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + )], + ) + .await; + assert_eq!( + producer_notify_count(&db).await, + 1, + "read-only → write transition must emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_duplicate_entries_use_persisted_state(db: Pool) { + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + )], + ) + .await; + reset_notify(&db).await; + + // Redeploy with conflicting duplicates for the same (kind, path), read-only + // first: ON CONFLICT DO NOTHING persists the read-only row and drops the + // write one, so the write producer is really gone → must emit (the diff must + // follow the persisted state, not the requested slice). + replace( + &db, + SP, + &[ + asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::R), + ), + asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + ), + ], + ) + .await; + assert_eq!( + producer_notify_count(&db).await, + 1, + "duplicate entries losing the persisted write producer must emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_duplicate_entries_keeping_write_does_not_emit(db: Pool) { + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + )], + ) + .await; + reset_notify(&db).await; + + // Same duplicates but write first: the write row persists, so the write set + // is unchanged → no emit. + replace( + &db, + SP, + &[ + asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::W), + ), + asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::R), + ), + ], + ) + .await; + assert_eq!( + producer_notify_count(&db).await, + 0, + "duplicate entries keeping the persisted write producer must not emit" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn replace_changing_only_readonly_does_not_emit(db: Pool) { + replace( + &db, + SP, + &[asset_at( + "u/test-user/res", + AssetKind::Resource, + Some(AssetUsageAccessType::R), + )], + ) + .await; + reset_notify(&db).await; + + // Swap one read-only producer for another: no write producer either side, so + // the write-set cache is untouched → no emit. + replace( + &db, + SP, + &[asset_at( + "u/test-user/other", + AssetKind::Resource, + Some(AssetUsageAccessType::R), + )], + ) + .await; + assert_eq!( + producer_notify_count(&db).await, + 0, + "changes confined to read-only producers must not emit" + ); +} diff --git a/backend/windmill-mcp/src/common/schema.rs b/backend/windmill-mcp/src/common/schema.rs index de90248689..96dcd023fc 100644 --- a/backend/windmill-mcp/src/common/schema.rs +++ b/backend/windmill-mcp/src/common/schema.rs @@ -193,6 +193,14 @@ pub fn enrich_resource_schemas( } } +/// The seven type names permitted by the JSON Schema draft 2020-12 `type` keyword. +fn is_valid_json_schema_type(t: &str) -> bool { + matches!( + t, + "null" | "boolean" | "object" | "array" | "number" | "string" | "integer" + ) +} + /// Transform a JSON schema for maximum MCP client compatibility. /// /// Ensures schemas conform to JSON Schema draft 2020-12 by: @@ -200,9 +208,12 @@ pub fn enrich_resource_schemas( /// - Removing invalid non-array `enum` values /// - Stripping non-standard keywords (`originalType`, `format` with `resource-*` prefix) /// - Rewriting the Windmill pseudo-type `type: "resource"` to `type: "string"` -/// - Fixing contradictory schemas (`type: "string"` with `properties` → `type: "object"`) +/// - Fixing contradictory schemas (`type: "string"` with `properties` → `type: "object"`, +/// or with `items` → `type: "array"`) +/// - Dropping invalid `type` values (e.g. the empty string `""` Windmill emits for +/// untyped fields) so the node validates as "any type" /// - Removing `default: null` when the type doesn't include `null` -/// - Adding `type: "object"` to empty schemas that have no type +/// - Adding `type: "object"` to property-bearing schemas that have no type pub fn make_schema_compatible(schema: &mut Value) { let Value::Object(obj) = schema else { return }; @@ -239,6 +250,55 @@ pub fn make_schema_compatible(schema: &mut Value) { } } + // 3b. Fix contradictory type: if `items` is present (and the node isn't an + // object), type must be "array". Windmill serializes a `list[...]` field as + // `type: "string"` with an `items` subschema; "string" + items is nonsense + // and leaves the element schema unreachable to the client. + if obj.contains_key("items") && !obj.contains_key("properties") { + match obj.get("type").and_then(|v| v.as_str()) { + Some("array") => {} + _ => { + obj.insert("type".to_string(), Value::String("array".to_string())); + } + } + } + + // 3c. Drop invalid `type` values. Windmill emits `type: ""` for fields + // declared without an explicit type; the empty string (and any other name + // outside the draft 2020-12 type enum) makes a strict validator reject the + // whole schema (e.g. Anthropic's tool registration). Removing it leaves the + // node untyped, which accepts any value -- the meaning of an untyped field. + match obj.get("type") { + None => {} + Some(Value::String(s)) => { + if !is_valid_json_schema_type(s) { + obj.remove("type"); + } + } + Some(Value::Array(arr)) => { + let filtered: Vec = arr + .iter() + .filter(|v| v.as_str().is_some_and(is_valid_json_schema_type)) + .cloned() + .collect(); + match filtered.len() { + 0 => { + obj.remove("type"); + } + 1 => { + obj.insert("type".to_string(), filtered.into_iter().next().unwrap()); + } + _ => { + obj.insert("type".to_string(), Value::Array(filtered)); + } + } + } + // `type` as null/number/bool/object is not a valid keyword value at all. + Some(_) => { + obj.remove("type"); + } + } + // 4. Convert integer to number if let Some(type_val) = obj.get_mut("type") { match type_val { @@ -273,11 +333,6 @@ pub fn make_schema_compatible(schema: &mut Value) { obj.remove("enum"); } - // 7. Ensure schemas with no type but with properties get type: "object" - if !obj.contains_key("type") && !obj.is_empty() { - obj.insert("type".to_string(), Value::String("object".to_string())); - } - // Recursively process nested schemas if let Some(Value::Object(props)) = obj.get_mut("properties") { for value in props.values_mut() { @@ -782,4 +837,137 @@ mod tests { json!("string") ); } + + /// Recursively assert no `type` keyword anywhere in the schema carries an + /// empty string or a name outside the draft 2020-12 type enum -- the exact + /// shape Anthropic rejects with "input_schema: JSON Schema is invalid". + fn assert_all_types_valid(node: &Value) { + if let Some(obj) = node.as_object() { + match obj.get("type") { + Some(Value::String(s)) => { + assert!(is_valid_json_schema_type(s), "invalid type string: {s:?}") + } + Some(Value::Array(arr)) => { + for t in arr { + let s = t.as_str().expect("type array entries must be strings"); + assert!(is_valid_json_schema_type(s), "invalid type entry: {s:?}"); + } + } + _ => {} + } + for v in obj.values() { + assert_all_types_valid(v); + } + } else if let Some(arr) = node.as_array() { + for v in arr { + assert_all_types_valid(v); + } + } + } + + #[test] + fn drops_empty_type_string() { + let mut schema = json!({ + "type": "object", + "properties": { + "value": { "type": "", "description": "" } + } + }); + + make_schema_compatible(&mut schema); + + // Empty type is removed entirely (untyped == accepts any value); it is + // NOT re-typed to object, so a scalar value still validates. + assert!(schema["properties"]["value"].get("type").is_none()); + assert_all_types_valid(&schema); + } + + #[test] + fn infers_array_type_from_items() { + let mut schema = json!({ + "type": "object", + "properties": { + "services": { + "type": "string", + "description": "An object parameter.", + "items": { "type": "object" } + } + } + }); + + make_schema_compatible(&mut schema); + + assert_eq!(schema["properties"]["services"]["type"], json!("array")); + assert_all_types_valid(&schema); + } + + #[test] + fn items_does_not_override_object_with_properties() { + // A node carrying both `properties` and a stray `items` is an object, + // not an array -- the object signal wins. + let mut schema = json!({ + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "items": { "type": "string" } + }); + + make_schema_compatible(&mut schema); + + assert_eq!(schema["type"], json!("object")); + } + + #[test] + fn filters_invalid_type_array_entries() { + let mut schema = json!({ "type": ["string", ""] }); + + make_schema_compatible(&mut schema); + + // Sole surviving entry collapses to a bare string. + assert_eq!(schema["type"], json!("string")); + } + + #[test] + fn drops_type_array_when_all_entries_invalid() { + let mut schema = json!({ "type": ["", "bogus"], "description": "x" }); + + make_schema_compatible(&mut schema); + + assert!(schema.get("type").is_none()); + } + + #[test] + fn customer_services_field_repro() { + // Repro of the reported failure: a `list[object]` script param that + // Windmill serialized as `type: "string"` + `items`, whose element + // object carried an untyped `value` field (`type: ""`). Anthropic + // rejected the whole tool list with + // "tools..custom.input_schema: JSON Schema is invalid". + let mut schema = json!({ + "type": "object", + "properties": { + "services": { + "items": { + "type": "object", + "properties": { + "serviceTypeId": { "description": "", "type": "string" }, + "value": { "description": "", "type": "" } + } + }, + "description": "An object parameter.", + "type": "string" + } + }, + "required": ["services"] + }); + + make_schema_compatible(&mut schema); + + assert_eq!(schema["properties"]["services"]["type"], json!("array")); + assert!(schema["properties"]["services"]["items"]["properties"]["value"] + .get("type") + .is_none()); + assert_all_types_valid(&schema); + } } diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index 252f7ace28..2f28a6d4c7 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -67,6 +67,13 @@ pub struct ClientWithScopes { pub allowed_domains: Option>, pub userinfo_url: Option, pub grant_types: Vec, + /// Resolved token endpoint, exposed so the connect dialog can prefill and + /// persist it on client-credentials accounts. + pub token_url: String, + /// Whether the instance entry carries shared credentials (non-empty id + + /// secret). Providers without them are bring-your-own only — the connect + /// dialog lists them under "Others", not "Instance-configured". + pub has_shared_credentials: bool, } /// Map of OAuth client names to their configurations @@ -81,6 +88,13 @@ pub struct OAuthConfig { pub token_url: String, pub userinfo_url: Option, pub scopes: Option>, + /// Default scopes for the client-credentials (2-legged) flow. These differ + /// from the authorization-code `scopes` for most providers (member/consent + /// scopes are invalid in a 2-legged token request), so CC never defaults to + /// `scopes`. Absent means no default scope — the caller supplies any + /// provider-specific scopes themselves. + #[serde(skip_serializing_if = "Option::is_none")] + pub cc_scopes: Option>, pub extra_params: Option>, pub extra_params_callback: Option>, pub req_body_auth: Option, @@ -91,10 +105,12 @@ pub struct OAuthConfig { /// entry, `build_oauth_clients` registers a second client under that key. #[serde(skip_serializing_if = "Option::is_none")] pub sandbox: Option, - /// Frontend-only metadata for per-instance OAuth providers (Snowflake, - /// ServiceNow, …) whose authorize/token URLs are derived from an - /// admin-entered instance name. Ignored by the backend, which only ever - /// sees the resulting concrete `connect_config`. + /// Metadata for per-instance OAuth providers (Snowflake, ServiceNow, Coupa, + /// …) whose authorize/token URLs carry an `{instance}` placeholder filled + /// from an instance name. The instance-settings UI uses it to build the + /// per-client `connect_config` for the authorization-code flow; the + /// client-credentials flow reads its `token_url`/`strip_suffix`/`label` + /// directly to host-pin the exchange. #[serde(skip_serializing_if = "Option::is_none")] pub connect_config_template: Option, } @@ -111,11 +127,13 @@ pub struct OAuthSandboxOverride { pub userinfo_url: Option, } -/// Frontend metadata for a per-instance OAuth provider. The instance-settings -/// UI renders one generic instance-name input and substitutes `{instance}` into +/// Metadata for a per-instance OAuth provider. The instance-settings UI renders +/// one generic instance-name input and substitutes `{instance}` into /// `auth_url`/`token_url` to build the per-client `connect_config`. Adding a new /// per-instance provider needs only a registry entry carrying this template — -/// no frontend code change. The backend never reads it. +/// no frontend code change. The client-credentials flow additionally reads +/// `token_url`, `strip_suffix`, and `label` from it server-side (see +/// `resolve_cc_token_url_input`) to host-pin the token exchange. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ConnectConfigTemplate { /// Properly-cased provider name for the settings dropdown (e.g. "ServiceNow"); @@ -126,10 +144,17 @@ pub struct ConnectConfigTemplate { pub placeholder: String, #[serde(skip_serializing_if = "Option::is_none")] pub help_url: Option, - pub auth_url: String, + /// Authorize endpoint (with `{instance}`). Absent for client-credentials-only + /// providers (e.g. Coupa) that have no browser sign-in flow. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auth_url: Option, pub token_url: String, #[serde(skip_serializing_if = "Option::is_none")] pub req_body_auth: Option, + /// Scopes copied into the built `connect_config` (e.g. NetSuite's + /// `rest_webservices`). Templated providers default to no scopes. + #[serde(skip_serializing_if = "Option::is_none")] + pub scopes: Option>, /// Key under `connect_config.extra_params` where the instance name is /// stored (defaults to `instance`). Snowflake uses `account_identifier` for /// backward compatibility with previously-saved configs. @@ -237,8 +262,12 @@ fn empty_string() -> String { "".to_string() } +/// Placeholder authorize URL for providers that only support the +/// client-credentials grant (the authorize endpoint is never used by it). +pub const MISSING_AUTH_URL: &str = "https://missing-auth-url"; + fn empty_auth() -> String { - "https://missing-auth-url".to_string() + MISSING_AUTH_URL.to_string() } fn default_grant_types() -> Vec { @@ -348,77 +377,349 @@ pub async fn build_slack_client( Ok(client) } -/// Build OAuth client for client credentials flow with resource-level credentials +/// Build OAuth client for client credentials flow with resource-level credentials. +/// +/// No instance-level entry is required: the provider endpoint config resolves +/// from the instance `oauths` entry when one exists, else from the static +/// registry, else is synthesized from the token URL override alone. Returns the +/// built client together with the resolved [`OAuthConfig`] so callers can reuse +/// its scopes / `extra_params_callback`. pub async fn build_client_credentials_oauth_client( db: &DB, client_name: &str, client_id: &str, client_secret: &str, - cc_token_url_override: Option<&str>, + resolved_token_url: Option<&str>, connect_configs_json: &str, -) -> error::Result<(OClient, OAuthClient)> { +) -> error::Result<(OClient, OAuthConfig)> { use windmill_common::global_settings::{load_value_from_global_settings, OAUTH_SETTING}; let oauths = load_value_from_global_settings(db, OAUTH_SETTING).await?; - let oauths = oauths.unwrap_or_default(); - let oauth_config = oauths - .get(client_name) - .ok_or_else(|| error::Error::BadRequest("OAuth configuration not found".to_string()))?; + let instance_entry: Option = oauths + .as_ref() + .and_then(|o| o.get(client_name)) + .and_then(|v| match serde_json::from_value(v.clone()) { + Ok(entry) => Some(entry), + Err(e) => { + tracing::warn!( + client = %client_name, + "Invalid instance OAuth entry, falling back to static registry: {e}" + ); + None + } + }); - let oauth_client_config: OAuthClient = serde_json::from_value(oauth_config.clone()) - .map_err(|e| error::Error::BadRequest(format!("Invalid OAuth config: {}", e)))?; - - let parse_static_configs = || { - serde_json::from_str::>(connect_configs_json).map_err(|e| { - error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)) - }) + let resolve_from_registry = |client_name: &str| -> error::Result> { + let static_configs = + serde_json::from_str::>(connect_configs_json).map_err( + |e| error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)), + )?; + Ok(resolve_registry_config(&static_configs, client_name)) }; - let resolve_from_registry = |client_name: &str| -> error::Result { - let static_configs = parse_static_configs()?; - resolve_registry_config(&static_configs, client_name).ok_or_else(|| { + + // A token URL alone is enough for client credentials: providers that only + // support this grant have no authorize endpoint to configure. + let instance_connect_config = instance_entry + .as_ref() + .and_then(|e| e.connect_config.clone()) + .filter(|c| !c.token_url.is_empty()) + .map(|mut c| { + if c.auth_url.is_empty() { + c.auth_url = empty_auth(); + } + c + }); + + let from_instance = instance_connect_config.is_some(); + let mut connect_config = match instance_connect_config { + Some(config) => config, + None => resolve_from_registry(client_name)?.ok_or_else(|| { error::Error::BadRequest(format!( - "OAuth configuration not found for '{}' in either global settings or static config", + "No token URL available for '{}': not found in instance OAuth settings or static \ + config", client_name )) - }) + })?, }; - let mut connect_config = if let Some(ref config) = oauth_client_config.connect_config { - if !config.auth_url.is_empty() && !config.token_url.is_empty() { - config.clone() - } else { - resolve_from_registry(client_name)? - } - } else { - resolve_from_registry(client_name)? - }; - - if let Some(override_url) = cc_token_url_override { - connect_config.token_url = override_url.to_string(); + // Registry providers default their client-credentials scopes from `cc_scopes`, + // never the authorization-code `scopes` (which several providers reject for a + // 2-legged token request). Instance-configured entries keep their admin-set + // scopes untouched. + if !from_instance { + connect_config.scopes = connect_config.cc_scopes.clone(); } + let caller_supplied_creds = !client_id.is_empty() && !client_secret.is_empty(); + + // Apply the server-resolved concrete token URL. Instance-templated providers + // (e.g. Coupa) carry an empty or `{instance}`-templated token URL in their + // registry config; the resolved value (host-pinned for bring-your-own, + // persisted on the row for refresh) is what completes it. The caller never + // supplies a free-form token URL: this value always comes from + // `resolve_cc_token_url_input` or a previously-resolved persisted URL. + if let Some(url) = resolved_token_url { + connect_config.token_url = url.to_string(); + } + if connect_config.token_url.is_empty() { + return Err(error::Error::BadRequest(format!( + "No token URL configured for '{}'", + client_name + ))); + } + + // Fall back to the instance entry's own credentials when the caller supplies + // none: the shared instance-level client-credentials setup, where an admin + // configures one service-account client for everyone and the secret never + // leaves the server. Only entries that explicitly enable the + // client_credentials grant qualify, so an authorization-code-only client's + // secret is never reused for this flow. + let instance_cc_creds = instance_entry.as_ref().filter(|e| { + e.grant_types.iter().any(|g| g == "client_credentials") + && !e.id.is_empty() + && !e.secret.is_empty() + }); + // All-or-nothing: use the caller's credentials only when both id and secret + // are present, otherwise fall back entirely to the instance entry. Never mix + // a caller-supplied id with the admin secret (or vice versa). + let (resolved_client_id, resolved_client_secret) = if caller_supplied_creds { + (client_id.to_string(), client_secret.to_string()) + } else { + instance_cc_creds + .map(|e| (e.id.clone(), e.secret.clone())) + .unwrap_or_default() + }; + let resource_oauth_client = OAuthClient { - id: client_id.to_string(), - secret: client_secret.to_string(), - allowed_domains: oauth_client_config.allowed_domains.clone(), + id: resolved_client_id, + secret: resolved_client_secret, + allowed_domains: instance_entry + .as_ref() + .and_then(|e| e.allowed_domains.clone()), connect_config: Some(connect_config.clone()), - login_config: oauth_client_config.login_config.clone(), - display_name: oauth_client_config.display_name.clone(), - grant_types: oauth_client_config.grant_types.clone(), - tenant: oauth_client_config.tenant.clone(), + login_config: instance_entry.as_ref().and_then(|e| e.login_config.clone()), + display_name: instance_entry.as_ref().and_then(|e| e.display_name.clone()), + grant_types: instance_entry + .as_ref() + .map(|e| e.grant_types.clone()) + .unwrap_or_else(default_grant_types), + tenant: instance_entry.as_ref().and_then(|e| e.tenant.clone()), }; let base_url = (**BASE_URL.load()).clone(); let (_, client) = build_basic_client( client_name.to_string(), - connect_config, + connect_config.clone(), resource_oauth_client, false, &base_url, None, )?; - Ok((client, oauth_client_config)) + Ok((client, connect_config)) +} + +/// Shared instance-level client-credentials for `client_name`: the `(id, secret, +/// token_url)` from its instance `oauths` entry, but only when that entry both +/// declares the `client_credentials` grant and carries non-empty credentials. +/// Lets the connect flow use one admin-configured service-account client instead +/// of asking each user for their own. +/// +/// # Authorization +/// Returns the admin's shared service-account secret, so callers MUST first +/// verify the caller's authorization to use it (workspace membership plus +/// read-write access — operators and read-only tokens are excluded). This helper +/// performs no authorization itself. +pub async fn resolve_instance_cc_credentials( + db: &DB, + client_name: &str, +) -> error::Result)>> { + use windmill_common::global_settings::{load_value_from_global_settings, OAUTH_SETTING}; + + let oauths = load_value_from_global_settings(db, OAUTH_SETTING).await?; + let entry: Option = oauths + .as_ref() + .and_then(|o| o.get(client_name)) + .and_then(|v| serde_json::from_value(v.clone()).ok()); + + Ok(entry.and_then(|e| { + let cc_grant = e.grant_types.iter().any(|g| g == "client_credentials"); + if cc_grant && !e.id.is_empty() && !e.secret.is_empty() { + // Token URL from the entry's connect_config (built by instance settings + // from the connect_config_template), so the account row is + // self-contained for refresh. + let token_url = e + .connect_config + .as_ref() + .map(|c| c.token_url.clone()) + .filter(|u| !u.is_empty()); + Some((e.id, e.secret, token_url)) + } else { + None + } + })) +} + +/// Resolve the concrete client-credentials token URL for a bring-your-own +/// connection. The caller never supplies a token URL: it always comes from the +/// built-in registry, so the exchange host can never be redirected. +/// +/// Supported only for registry providers. For one whose CC token URL carries an +/// `{instance}` placeholder (Coupa, ServiceNow, …) — declared in its +/// `connect_config_template` — the caller supplies only an instance name, +/// validated as a bare hostname label and substituted into the fixed-host +/// template. A fixed-host registry provider uses its registry token URL directly. +/// A custom resource type (no registry entry) is rejected: there is no known host +/// to send credentials to. +pub fn resolve_cc_token_url_input( + connect_configs_json: &str, + client_name: &str, + caller_instance: Option<&str>, +) -> error::Result { + let Some(cfg) = serde_json::from_str::>(connect_configs_json) + .ok() + .and_then(|m| resolve_registry_config(&m, client_name)) + else { + return Err(error::Error::BadRequest(format!( + "Client credentials with your own credentials are only supported for built-in OAuth \ + providers, not '{client_name}'. Configure shared credentials on the instance OAuth \ + entry instead." + ))); + }; + + // Instance-templated providers carry the `{instance}` token URL (and its + // label/strip_suffix) in `connect_config_template`; fixed-host providers use + // the plain `token_url`. + let tmpl = cfg.connect_config_template.as_ref(); + let template = tmpl + .map(|t| t.token_url.clone()) + .filter(|u| !u.is_empty()) + .or_else(|| Some(cfg.token_url.clone()).filter(|u| !u.is_empty())) + .ok_or_else(|| { + error::Error::BadRequest(format!("No token URL is configured for '{client_name}'")) + })?; + + if !template.contains("{instance}") { + // Fixed-host registry provider: its registry token URL is authoritative. + return Ok(template); + } + + // Structural host-pinning guard: only substitute when `{instance}` is the + // leftmost host label of a fixed-host template (`scheme://{instance}.fixed-host/…`). + // The hostname-label validation below keeps the value clean, but only this + // check guarantees the substituted value can never change the registrable + // domain — so a malformed template (e.g. `https://{instance}/token`) can't turn + // the caller's instance name into a full attacker-controlled host (SSRF / + // credential exfiltration). The template is a code-reviewed registry file, so a + // violation is a programming error. + let placeholder = "{instance}"; + let idx = template.find(placeholder).unwrap(); + let after = &template[idx + placeholder.len()..]; + if !template[..idx].ends_with("://") || !after.starts_with('.') { + return Err(error::Error::InternalErr(format!( + "Invalid instance-templated token URL for '{client_name}': {{instance}} must be the \ + leftmost host label (scheme://{{instance}}.fixed-host/…)" + ))); + } + + let raw = caller_instance + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + error::Error::BadRequest(format!( + "{} is required for {client_name}", + tmpl.map(|t| t.label.as_str()).unwrap_or("An instance name") + )) + })?; + // Strip an optional known host suffix so the user can paste a full host or a + // bare name, then accept only a hostname label — never any character that + // could move the host out of the template's domain. + let value = tmpl + .and_then(|t| t.strip_suffix.as_deref()) + .and_then(|sfx| raw.strip_suffix(sfx)) + .unwrap_or(raw) + .trim_end_matches('.'); + let valid = !value.is_empty() + && !value.starts_with(['-', '.']) + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'.'); + if !valid { + return Err(error::Error::BadRequest(format!( + "invalid instance name '{raw}' for {client_name}" + ))); + } + Ok(template.replace("{instance}", value)) +} + +/// Resolve the concrete bring-your-own client-credentials token URL for any +/// provider, never from a caller-supplied URL: +/// - **Built-in registry providers** resolve from the registry via +/// [`resolve_cc_token_url_input`] (host-pinned from the caller's instance name +/// for instance-templated ones). +/// - **Custom providers configured at the instance level** use the admin's +/// `connect_config.token_url`. The caller has no instance template to fill, so +/// an instance name is rejected. +/// +/// This is the single entry point the connect/account-creation handlers should +/// use so both resolve identically. +pub async fn resolve_cc_token_url( + db: &DB, + client_name: &str, + caller_instance: Option<&str>, + connect_configs_json: &str, +) -> error::Result { + use windmill_common::global_settings::{load_value_from_global_settings, OAUTH_SETTING}; + + let supports_cc = + |grant_types: &[String]| grant_types.iter().any(|g| g == "client_credentials"); + + let registry_cfg = serde_json::from_str::>(connect_configs_json) + .ok() + .and_then(|m| resolve_registry_config(&m, client_name)); + if let Some(cfg) = registry_cfg { + // Built-in provider: only honor it for client credentials if it actually + // declares that grant, so an authorization-code-only provider can't be + // driven through the CC API. + if !supports_cc(&cfg.grant_types) { + return Err(error::Error::BadRequest(format!( + "'{client_name}' is not enabled for the client_credentials grant" + ))); + } + return resolve_cc_token_url_input(connect_configs_json, client_name, caller_instance); + } + + // Custom (non-registry) provider: the token URL comes from the admin's + // instance connect_config (an admin-configured, trusted host), never the + // caller. The instance entry must also enable the client-credentials grant. + let entry: Option = load_value_from_global_settings(db, OAUTH_SETTING) + .await? + .as_ref() + .and_then(|o| o.get(client_name)) + .and_then(|v| serde_json::from_value(v.clone()).ok()); + let instance_token_url = entry + .as_ref() + .filter(|e| supports_cc(&e.grant_types)) + .and_then(|e| e.connect_config.clone()) + .map(|c| c.token_url) + .filter(|u| !u.is_empty()); + match instance_token_url { + Some(_) + if caller_instance + .map(|s| !s.trim().is_empty()) + .unwrap_or(false) => + { + Err(error::Error::BadRequest(format!( + "An instance name only applies to built-in instance-templated providers, not \ + '{client_name}'" + ))) + } + Some(url) => Ok(url), + None => Err(error::Error::BadRequest(format!( + "Client credentials with your own credentials require '{client_name}' to be a built-in \ + OAuth provider or an instance entry that enables the client_credentials grant" + ))), + } } /// Exchange authorization code for tokens @@ -462,7 +763,7 @@ pub async fn exchange_token( client: OClient, refresh_token: &str, grant_type: &str, - oauth_client_info: Option<&ClientWithScopes>, + extra_params_callback: Option<&HashMap>, http_client: &reqwest::Client, scopes: Option<&[String]>, ) -> Result { @@ -483,11 +784,9 @@ pub async fn exchange_token( "client_credentials" => { let mut token_request = client.exchange_client_credentials(); - if let Some(oauth_info) = oauth_client_info { - if let Some(extra_params) = oauth_info.extra_params_callback.as_ref() { - for (key, value) in extra_params.iter() { - token_request = token_request.param(key.clone(), value.clone()); - } + if let Some(extra_params) = extra_params_callback { + for (key, value) in extra_params.iter() { + token_request = token_request.param(key.clone(), value.clone()); } } @@ -579,49 +878,78 @@ pub async fn refresh_token_for_account<'c>( http_client: &reqwest::Client, connect_configs_json: &str, ) -> error::Result { - let oauth_client_info = oauth_clients - .connects - .get(&account.client) - .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))? - .clone(); + // Instance-configured client: required for authorization_code (the refresh + // token exchange uses the instance app's credentials). For client_credentials + // it is resolved inside `build_client_credentials_oauth_client` instead. + let oauth_client_info = oauth_clients.connects.get(&account.client).cloned(); - let mut client = if account.grant_type == "client_credentials" { - match (&account.cc_client_id, &account.cc_client_secret) { - (Some(client_id), Some(client_secret)) => { - let (client, _) = build_client_credentials_oauth_client( - db, - &account.client, - client_id, - client_secret, - account.cc_token_url.as_deref(), - connect_configs_json, - ) - .await?; - client - } - _ => { - return Err(error::Error::BadRequest( - "client_credentials flow requires cc_client_id and cc_client_secret to be stored in account".to_string() - )); - } - } + let is_client_credentials = account.grant_type == "client_credentials"; + + let (mut client, cc_config) = if is_client_credentials { + // Bring-your-own accounts store their own credentials (and resolved token + // URL) on the row. Shared instance accounts store none: passing empty + // credentials makes the builder re-resolve the admin's service-account + // credentials and token URL from the instance entry on every refresh, so a + // rotated or removed shared secret takes effect immediately (mirrors the + // authorization-code model, where the row never holds the app secret). + let (client_id, client_secret) = match (&account.cc_client_id, &account.cc_client_secret) { + (Some(id), Some(secret)) => (id.as_str(), secret.as_str()), + _ => ("", ""), + }; + let (client, config) = build_client_credentials_oauth_client( + db, + &account.client, + client_id, + client_secret, + account.cc_token_url.as_deref(), + connect_configs_json, + ) + .await?; + (client, Some(config)) } else { - oauth_client_info.client.to_owned() + let info = oauth_client_info + .as_ref() + .ok_or_else(|| error::Error::BadRequest("invalid client".to_string()))?; + (info.client.to_owned(), None) }; - // Account-level scopes override instance-level scopes + // Account-level scopes (when stored) override these defaults. Client-credentials + // accounts default to the resolved CC config's scopes (`cc_scopes` for registry + // providers, the admin's instance scopes for custom ones) — never the instance + // client's authorization-code scopes, which are invalid in a 2-legged request. + // Authorization-code accounts default to the instance client's scopes. + let fallback_scopes = if is_client_credentials { + cc_config + .as_ref() + .and_then(|c| c.scopes.clone()) + .unwrap_or_default() + } else { + oauth_client_info + .as_ref() + .map(|i| i.scopes.clone()) + .unwrap_or_default() + }; let effective_scopes = account .scopes .as_deref() .filter(|s| !s.is_empty()) - .unwrap_or(&oauth_client_info.scopes); + .unwrap_or(&fallback_scopes); - if account.grant_type == "client_credentials" { + if is_client_credentials { for scope in effective_scopes.iter() { client.add_scope(scope); } } + let extra_params_callback = oauth_client_info + .as_ref() + .and_then(|i| i.extra_params_callback.clone()) + .or_else(|| { + cc_config + .as_ref() + .and_then(|c| c.extra_params_callback.clone()) + }); + tracing::info!( grant_type = %account.grant_type, client = %account.client, @@ -634,7 +962,7 @@ pub async fn refresh_token_for_account<'c>( client, &account.refresh_token, &account.grant_type, - Some(&oauth_client_info), + extra_params_callback.as_ref(), http_client, Some(effective_scopes), ) @@ -846,6 +1174,7 @@ mod tests { token_url: "https://account.example.com/oauth/token".to_string(), userinfo_url: Some("https://account.example.com/userinfo".to_string()), scopes: Some(vec!["signature".to_string()]), + cc_scopes: None, extra_params: None, extra_params_callback: None, req_body_auth: None, @@ -927,4 +1256,98 @@ mod tests { registry.insert("docusign".to_string(), sample_oauth_config(false)); assert!(resolve_registry_config(®istry, "docusign_sandbox").is_none()); } + + const CC_REGISTRY: &str = r#"{ + "coupa": { + "grant_types": ["client_credentials"], + "connect_config_template": { + "label": "Coupa instance", + "placeholder": "x", + "token_url": "https://{instance}.coupahost.com/oauth2/token", + "strip_suffix": ".coupahost.com" + } + }, + "servicenow": { + "grant_types": ["authorization_code", "client_credentials"], + "connect_config_template": { + "label": "ServiceNow instance", + "placeholder": "dev12345", + "auth_url": "https://{instance}.service-now.com/oauth_auth.do", + "token_url": "https://{instance}.service-now.com/oauth_token.do", + "strip_suffix": ".service-now.com" + } + }, + "visma": { + "auth_url": "https://connect.visma.com/connect/authorize", + "token_url": "https://connect.visma.com/connect/token", + "grant_types": ["authorization_code", "client_credentials"] + }, + "bad_host_tpl": { + "grant_types": ["client_credentials"], + "connect_config_template": { + "label": "x", "placeholder": "x", + "token_url": "https://{instance}/token" + } + }, + "bad_mid_tpl": { + "grant_types": ["client_credentials"], + "connect_config_template": { + "label": "x", "placeholder": "x", + "token_url": "https://api.{instance}.evil.com/token" + } + } + }"#; + + #[test] + fn cc_token_url_templated_substitutes_instance() { + let url = resolve_cc_token_url_input(CC_REGISTRY, "coupa", Some("acme")).unwrap(); + assert_eq!(url, "https://acme.coupahost.com/oauth2/token"); + } + + #[test] + fn cc_token_url_templated_from_connect_config_template() { + // ServiceNow's CC token URL comes from its connect_config_template. + let url = resolve_cc_token_url_input(CC_REGISTRY, "servicenow", Some("dev99")).unwrap(); + assert_eq!(url, "https://dev99.service-now.com/oauth_token.do"); + } + + #[test] + fn cc_token_url_strips_known_host_suffix() { + let url = + resolve_cc_token_url_input(CC_REGISTRY, "coupa", Some("acme.coupahost.com")).unwrap(); + assert_eq!(url, "https://acme.coupahost.com/oauth2/token"); + } + + #[test] + fn cc_token_url_rejects_instance_that_escapes_the_host() { + // A '/' (or any non-hostname char) must not let the caller move the host + // out of the template's domain. + assert!(resolve_cc_token_url_input(CC_REGISTRY, "coupa", Some("evil.com/oauth")).is_err()); + assert!(resolve_cc_token_url_input(CC_REGISTRY, "coupa", Some("a@b")).is_err()); + } + + #[test] + fn cc_token_url_requires_instance_when_templated() { + assert!(resolve_cc_token_url_input(CC_REGISTRY, "coupa", None).is_err()); + } + + #[test] + fn cc_token_url_fixed_host_uses_registry_url() { + let url = resolve_cc_token_url_input(CC_REGISTRY, "visma", None).unwrap(); + assert_eq!(url, "https://connect.visma.com/connect/token"); + } + + #[test] + fn cc_token_url_rejects_custom_provider() { + // No registry entry: bring-your-own client credentials are not allowed. + assert!(resolve_cc_token_url_input(CC_REGISTRY, "my_custom_thing", Some("acme")).is_err()); + } + + #[test] + fn cc_token_url_rejects_template_not_in_subdomain_position() { + // `{instance}` must be the leftmost host label of a fixed-host template, so + // a malformed template can't let the instance value control the host. + assert!(resolve_cc_token_url_input(CC_REGISTRY, "bad_host_tpl", Some("evil.com")).is_err()); + assert!(resolve_cc_token_url_input(CC_REGISTRY, "bad_mid_tpl", Some("evil")).is_err()); + } } diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 4b1279609f..cf30efe116 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -535,7 +535,134 @@ pub fn build_filesystem_client(root_path: &str) -> error::Result) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[cfg(feature = "parquet")] +#[async_trait] +impl ObjectStore for FilesystemStoreIgnoringAttributes { + async fn put_opts( + &self, + location: &object_store::path::Path, + payload: object_store::PutPayload, + mut opts: object_store::PutOptions, + ) -> object_store::Result { + opts.attributes = Default::default(); + self.0.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &object_store::path::Path, + mut opts: object_store::PutMultipartOpts, + ) -> object_store::Result> { + opts.attributes = Default::default(); + self.0.put_multipart_opts(location, opts).await + } + + async fn get_opts( + &self, + location: &object_store::path::Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.0.get_opts(location, options).await + } + + async fn get_range( + &self, + location: &object_store::path::Path, + range: std::ops::Range, + ) -> object_store::Result { + self.0.get_range(location, range).await + } + + async fn get_ranges( + &self, + location: &object_store::path::Path, + ranges: &[std::ops::Range], + ) -> object_store::Result> { + self.0.get_ranges(location, ranges).await + } + + async fn head( + &self, + location: &object_store::path::Path, + ) -> object_store::Result { + self.0.head(location).await + } + + async fn delete(&self, location: &object_store::path::Path) -> object_store::Result<()> { + self.0.delete(location).await + } + + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.0.list(prefix) + } + + fn list_with_offset( + &self, + prefix: Option<&object_store::path::Path>, + offset: &object_store::path::Path, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.0.list_with_offset(prefix, offset) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.0.list_with_delimiter(prefix).await + } + + async fn copy( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.copy(from, to).await + } + + async fn rename( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.rename(from, to).await + } + + async fn copy_if_not_exists( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.copy_if_not_exists(from, to).await + } + + async fn rename_if_not_exists( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.rename_if_not_exists(from, to).await + } } #[cfg(feature = "parquet")] diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs new file mode 100644 index 0000000000..7647ffc864 --- /dev/null +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -0,0 +1,736 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Runtime fan-out for asset-triggered scripts. +//! +//! When a script writes an asset and a downstream script subscribes to +//! that asset via `// on s3://...`, this module pushes a job for each +//! subscriber after the producer's job completes successfully. Any +//! asset-writing top-level script cascades — there is no `// pipeline` +//! gate on the producer side; subscriptions alone define the graph. +//! +//! Eligibility (V1, narrow on purpose): +//! - Producer kind is `Script` or `Preview`. Flows defer. +//! - Producer is top-level (no `parent_job`, no `flow_step_id`). +//! - Producer succeeded. +//! - The producer's args do not contain `_wmill_skip_asset_dispatch: true`. +//! +//! Subscribers (V1): +//! - Only `script` runnables. Flow subscribers defer. +//! - The subscriber must have at least one non-archived script row. +//! - A subscriber is skipped if its path equals the producer's path +//! (self-loop) or already appears in the cascade lineage +//! (`trigger.chain`) — cycle detection, which bounds the cascade +//! without capping legitimate depth. +//! +//! Args sent to subscribers: +//! ```json +//! { +//! "trigger": { +//! "kind": "asset", +//! "asset_kind": "s3object", +//! "asset_path": "...", +//! "producer_path": "...", +//! "producer_job_id": "...", +//! "chain": ["f/a/producer0", "f/a/producer1"] +//! } +//! } +//! ``` +//! +//! Errors are logged but never bubble up to fail the producer's job. + +use crate::{push, MiniCompletedJob, PushArgs, PushIsolationLevel}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::{Pool, Postgres}; +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; +use windmill_common::assets::AssetKind; +use windmill_common::error::{self, Result}; +use windmill_common::get_latest_deployed_hash_for_path; +use windmill_common::jobs::{JobKind, JobPayload, JobTriggerKind}; +use windmill_common::partition::PARTITION_ARG; +use windmill_common::scripts::ScriptHash; +use windmill_common::triggers::TriggerMetadata; +use windmill_common::users::{get_email_from_permissioned_as, username_to_permissioned_as}; +use windmill_common::worker::to_raw_value; +use windmill_common::DB; + +/// Reserved arg key that suppresses asset-trigger dispatch for a single run. +/// Set by the test panel when the user opts out of the cascade. +pub const SKIP_ASSET_DISPATCH_ARG: &str = "_wmill_skip_asset_dispatch"; + +/// Arg key holding the cascade trigger object (carries `chain`, `partition`, +/// producer metadata) injected into every dispatched subscriber. +const TRIGGER_ARG: &str = "trigger"; + +/// Arg key (under `trigger.chain`) carrying the cascade lineage: the ordered +/// list of producer paths already run in this chain. Used to detect cycles +/// (a producer re-appearing) and stop only the cyclic edge — so deep but +/// *acyclic* pipelines are never truncated. +const CHAIN_KEY: &str = "chain"; + +/// Safety backstop on lineage length. Cycle detection already bounds an +/// acyclic cascade (a path can't repeat), so this only guards against a +/// runaway from a bug. Set far above any real pipeline depth. +const MAX_CHAIN_LEN: usize = 1000; + +/// Returned to the caller (the worker's completed-job hook) so logs can +/// reference the dispatched ids. +#[derive(Debug, Default)] +pub struct DispatchResult { + pub dispatched: Vec, +} + +/// Per-decision outcome persisted to `dispatch_event` so the producer's +/// job detail page can show what happened to each subscriber. Mirrors +/// the `DISPATCH_OUTCOME` Postgres enum exactly. +#[derive(Debug, Clone, Copy, sqlx::Type)] +#[sqlx(type_name = "DISPATCH_OUTCOME", rename_all = "snake_case")] +enum DispatchOutcome { + Dispatched, + JoinPending, + Skipped, +} + +/// Outcome-specific fields. Event constructors take the four "always-present" +/// columns positionally and bundle the rest here so each call site only +/// names what it actually carries. +#[derive(Debug, Default)] +struct EventOptions<'a> { + child_job_id: Option, + partition: Option<&'a str>, + received_inputs: Option, + required_inputs: Option, + debounce_s: Option, + reason: Option<&'a str>, +} + +/// One accumulated `dispatch_event` row. Owned (not borrowed) so the whole +/// dispatch pass can collect rows and flush them in a single batched INSERT +/// at the end, avoiding an N+1 (one INSERT per subscriber × asset write). +#[derive(Debug)] +struct EventRow { + subscriber_path: String, + asset_kind: AssetKind, + asset_path: String, + outcome: DispatchOutcome, + child_job_id: Option, + partition: Option, + received_inputs: Option, + required_inputs: Option, + debounce_s: Option, + reason: Option, +} + +impl EventRow { + fn new( + subscriber_path: &str, + asset_kind: AssetKind, + asset_path: &str, + outcome: DispatchOutcome, + opts: EventOptions<'_>, + ) -> Self { + EventRow { + subscriber_path: subscriber_path.to_string(), + asset_kind, + asset_path: asset_path.to_string(), + outcome, + child_job_id: opts.child_job_id, + partition: opts.partition.map(str::to_string), + received_inputs: opts.received_inputs, + required_inputs: opts.required_inputs, + debounce_s: opts.debounce_s, + reason: opts.reason.map(str::to_string), + } + } +} + +/// Best-effort batched insert into `dispatch_event`. Never propagates — the +/// dispatch contract is "logging failures must not retroactively fail the +/// producer's job." All rows accumulated over a dispatch pass go in one +/// INSERT (UNNEST) to avoid an N+1 across (subscriber × asset write). +async fn flush_events(db: &DB, workspace_id: &str, producer_job_id: Uuid, events: &[EventRow]) { + if events.is_empty() { + return; + } + // Column-oriented arrays for UNNEST. Each Vec is one column across all rows. + let subscriber_paths: Vec = events.iter().map(|e| e.subscriber_path.clone()).collect(); + let asset_kinds: Vec = events.iter().map(|e| e.asset_kind).collect(); + let asset_paths: Vec = events.iter().map(|e| e.asset_path.clone()).collect(); + let outcomes: Vec = events.iter().map(|e| e.outcome).collect(); + let child_job_ids: Vec> = events.iter().map(|e| e.child_job_id).collect(); + let partitions: Vec> = events.iter().map(|e| e.partition.clone()).collect(); + let received_inputs: Vec> = events.iter().map(|e| e.received_inputs).collect(); + let required_inputs: Vec> = events.iter().map(|e| e.required_inputs).collect(); + let debounce_s: Vec> = events.iter().map(|e| e.debounce_s).collect(); + let reasons: Vec> = events.iter().map(|e| e.reason.clone()).collect(); + + let res = sqlx::query!( + r#"INSERT INTO dispatch_event ( + workspace_id, producer_job_id, subscriber_path, + asset_kind, asset_path, outcome, + child_job_id, partition, + received_inputs, required_inputs, + debounce_s, reason + ) + SELECT $1, $2, sp, ak, ap, oc, cj, pt, ri, rq, db, rs + FROM unnest( + $3::text[], $4::ASSET_KIND[], $5::text[], $6::DISPATCH_OUTCOME[], + $7::uuid[], $8::text[], $9::int[], $10::int[], $11::int[], $12::text[] + ) AS t(sp, ak, ap, oc, cj, pt, ri, rq, db, rs)"#, + workspace_id, + producer_job_id, + &subscriber_paths, + asset_kinds as Vec, + &asset_paths, + outcomes as Vec, + &child_job_ids as &[Option], + &partitions as &[Option], + &received_inputs as &[Option], + &required_inputs as &[Option], + &debounce_s as &[Option], + &reasons as &[Option], + ) + .execute(db) + .await; + if let Err(e) = res { + tracing::error!( + "failed to record {} dispatch_event row(s) for producer {}: {e:#}", + events.len(), + producer_job_id + ); + } +} + +/// Top-level entry. Returns `Ok(default)` and logs on any internal failure +/// rather than propagating, because dispatch is best-effort and must not +/// retroactively fail the producer. +pub async fn dispatch_asset_triggers(db: &DB, job: &MiniCompletedJob) -> DispatchResult { + match try_dispatch(db, job).await { + Ok(r) => r, + Err(e) => { + tracing::error!("asset-trigger dispatch failed for job {}: {e:#}", job.id); + DispatchResult::default() + } + } +} + +async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result { + if !is_eligible_kind(job) { + return Ok(DispatchResult::default()); + } + let runnable_path = match job.runnable_path.as_deref() { + Some(p) if !p.is_empty() => p, + _ => return Ok(DispatchResult::default()), + }; + + // Producer gate (cached): this hook fires on every top-level + // script/preview completion, and the overwhelmingly common case is a + // script that writes no asset. The per-workspace producer→writes map is + // cached and invalidated by a trigger on `asset`, so a non-producer + // completion costs one in-memory lookup and zero queries. The map is + // keyed on the deploy-time `asset` table by path, so an undeployed/new + // preview (no asset rows for its path) is a non-producer and never + // cascades — same as the previous per-completion lookup. + let producers = workspace_producer_writes(db, &job.workspace_id).await?; + let Some(writes) = producers.get(runnable_path).cloned() else { + return Ok(DispatchResult::default()); + }; + + let args = fetch_args(db, &job.workspace_id, job.id).await?; + if read_skip_arg(args.as_ref()) { + return Ok(DispatchResult::default()); + } + // Parse the cascade `trigger` object once; both the lineage chain and + // the propagated partition are read from it. + let trigger_map = args + .as_ref() + .and_then(|a| a.get(TRIGGER_ARG)) + .and_then(|t| serde_json::from_str::>>(t.get()).ok()); + let chain = read_chain(trigger_map.as_ref()); + let partition = read_partition(args.as_ref(), trigger_map.as_ref()); + if chain.len() >= MAX_CHAIN_LEN { + tracing::warn!( + "asset-trigger dispatch skipped: cascade lineage length {} >= backstop {} (job {}, path {})", + chain.len(), + MAX_CHAIN_LEN, + job.id, + runnable_path + ); + return Ok(DispatchResult::default()); + } + // Lineage propagated to any subscriber pushed from this producer: the + // ancestors that already ran, plus this producer. + let mut next_chain = chain.clone(); + next_chain.push(runnable_path.to_string()); + + let mut dispatched = Vec::new(); + // Best-effort dispatch_event rows accumulated over the whole pass and + // flushed in one batched INSERT at the end (avoids an N+1 over + // subscriber × asset write). The mid-pass join-slot writes + // (record_and_check_join_slot) are a separate table and unaffected. + let mut events: Vec = Vec::new(); + for (asset_kind, asset_path) in writes { + let Some(prefix) = asset_kind.canonical_prefix() else { + continue; + }; + let trigger_ref = format!("{}{}", prefix, asset_path); + let subs = fetch_subscribers(db, &job.workspace_id, &trigger_ref).await?; + for sub in subs { + let Subscriber { path: sub_path, join_all, debounce_s, retry_count, retry_delay_s } = + sub; + if sub_path == runnable_path { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some("self_loop"), ..Default::default() }, + )); + continue; + } + // Cycle guard: a subscriber already in this producer's lineage + // would re-enter the chain (A→…→A), looping forever. Stop only + // this edge — sibling branches still dispatch, and acyclic chains + // of any depth are unaffected. + if chain.iter().any(|p| p == &sub_path) { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some("cycle_detected"), ..Default::default() }, + )); + continue; + } + if join_all { + match crate::cascade::handle_join( + db, + &job.workspace_id, + &sub_path, + &trigger_ref, + partition.as_deref(), + ) + .await + { + Ok(crate::cascade::JoinDecision::Skip(reason)) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some(reason), ..Default::default() }, + )); + continue; + } + Ok(crate::cascade::JoinDecision::Pending { received, required }) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::JoinPending, + EventOptions { + partition: partition.as_deref(), + received_inputs: Some(received), + required_inputs: Some(required), + ..Default::default() + }, + )); + continue; // slot incomplete — wait for the rest + } + Ok(crate::cascade::JoinDecision::Fire) => {} // fall through to push + Err(e) => { + tracing::error!("join-slot check failed for {}: {e:#}", sub_path); + continue; + } + } + } + match push_subscriber( + db, + job, + &sub_path, + asset_kind, + &asset_path, + runnable_path, + &next_chain, + partition.as_deref(), + debounce_s, + retry_count, + retry_delay_s, + ) + .await + { + Ok(id) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Dispatched, + EventOptions { + child_job_id: Some(id), + partition: partition.as_deref(), + debounce_s, + ..Default::default() + }, + )); + dispatched.push(id); + } + Err(e) => { + tracing::error!("failed to push asset-triggered job for {}: {e:#}", sub_path) + } + } + } + } + + flush_events(db, &job.workspace_id, job.id, &events).await; + + if !dispatched.is_empty() { + tracing::info!( + "asset-trigger dispatch from job {} ({}): pushed {} downstream jobs", + job.id, + runnable_path, + dispatched.len() + ); + } + Ok(DispatchResult { dispatched }) +} + +fn is_eligible_kind(job: &MiniCompletedJob) -> bool { + if !matches!(job.kind, JobKind::Script | JobKind::Preview) { + return false; + } + if job.parent_job.is_some() || job.flow_step_id.is_some() { + return false; + } + true +} + +async fn fetch_args( + db: &Pool, + workspace_id: &str, + job_id: Uuid, +) -> Result>>> { + // Read from v2_job because args live there permanently — v2_job_completed + // is the *result* row and doesn't carry args. The producer's v2_job row + // is still present at dispatch time (deletion happens later in the + // completion pipeline, after this hook). + let row = sqlx::query!( + r#"SELECT args AS "args!: Json>>" + FROM v2_job + WHERE workspace_id = $1 AND id = $2"#, + workspace_id, + job_id, + ) + .fetch_optional(db) + .await?; + Ok(row.map(|r| r.args.0)) +} + +fn read_skip_arg(args: Option<&HashMap>>) -> bool { + args.and_then(|a| a.get(SKIP_ASSET_DISPATCH_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or(false) +} + +fn read_chain(trigger_map: Option<&HashMap>>) -> Vec { + trigger_map + .and_then(|m| m.get(CHAIN_KEY)) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default() +} + +/// The partition value the producer ran with, if any. Resolved once at the +/// top of a chain (run-start) and threaded down here so every cascaded job +/// materializes the same partition without re-resolving. Top-level +/// `partition` arg (run-start injection) takes precedence over the +/// `trigger.partition` carried from an upstream cascade hop. +fn read_partition( + args: Option<&HashMap>>, + trigger_map: Option<&HashMap>>, +) -> Option { + if let Some(v) = args.and_then(|a| a.get(PARTITION_ARG)) { + if let Ok(s) = serde_json::from_str::(v.get()) { + return Some(s); + } + } + serde_json::from_str::(trigger_map?.get(PARTITION_ARG)?.get()).ok() +} + +lazy_static::lazy_static! { + /// Per-workspace map of producer script path → the assets it writes + /// (`usage_access_type IN ('w','rw')`). Serves both the producer gate + /// (is this path a producer?) and the writes themselves, so a completion + /// that isn't a producer costs a single in-memory lookup and zero + /// queries — the dispatch hook fires on every top-level script/preview + /// completion instance-wide, the overwhelming majority of which write no + /// asset. An empty map means the workspace has no asset producers (no + /// pipelines). Invalidated per workspace by `notify_asset_producer_change` + /// (a trigger on `asset`) through the polling notify system; until the + /// next poll a freshly-deployed producer may not cascade (sub-poll lag, + /// acceptable for a data pipeline). + pub static ref ASSET_PRODUCER_WRITES_CACHE: + quick_cache::sync::Cache>>> = + quick_cache::sync::Cache::new(1000); +} + +/// Test hook: disables the producer-writes cache so every dispatch reads the +/// current DB. Integration tests use `#[sqlx::test]` isolated DBs that all +/// share one workspace id, so a process-global cache keyed by workspace would +/// clobber across DBs under concurrent test threads. Always `false` in +/// production (the cache is invalidated via the notify_event poller instead). +pub static ASSET_PRODUCER_CACHE_DISABLED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Load (cached) the producer→writes map for a workspace. The single load +/// query replaces the per-completion producer lookup; once cached, every +/// completion in the workspace is served from memory until invalidation. +async fn workspace_producer_writes( + db: &Pool, + workspace_id: &str, +) -> Result>>> { + let use_cache = !ASSET_PRODUCER_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + if use_cache { + if let Some(map) = ASSET_PRODUCER_WRITES_CACHE.get(workspace_id) { + return Ok(map); + } + } + let rows = sqlx::query!( + r#" + SELECT + usage_path AS "usage_path!", + kind AS "kind!: AssetKind", + path AS "path!" + FROM asset + WHERE workspace_id = $1 + AND usage_kind = 'script' + AND usage_access_type IN ('w', 'rw') + "#, + workspace_id, + ) + .fetch_all(db) + .await?; + let mut map: HashMap> = HashMap::new(); + for r in rows { + map.entry(r.usage_path).or_default().push((r.kind, r.path)); + } + let map = Arc::new(map); + if use_cache { + ASSET_PRODUCER_WRITES_CACHE.insert(workspace_id.to_string(), map.clone()); + } + Ok(map) +} + +/// A subscriber row resolved from `script_trigger`. Bundles the per-edge +/// options (debounce) and the script-level policy fields (`join_all`, +/// retry) that travel together to dispatch. +struct Subscriber { + path: String, + join_all: bool, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +} + +async fn fetch_subscribers( + db: &Pool, + workspace_id: &str, + trigger_ref: &str, +) -> Result> { + // V1: script subscribers only. Flow subscribers (`runnable_kind = 'flow'`) + // are intentionally excluded — wiring them is straightforward but the + // payload shape and permissioning need their own pass. + // `join_all` = `// trigger all` (AND join); `debounce_s` = the opt-in + // debounce window resolved at deploy (NULL = fan-out, the default). + // `retry_count` / `retry_delay_s` = the `// retry []` policy + // (NULL = no retry). + let rows = sqlx::query!( + r#" + SELECT runnable_path AS "runnable_path!", join_all AS "join_all!", debounce_s, + retry_count, retry_delay_s + FROM script_trigger + WHERE workspace_id = $1 + AND trigger_kind = 'asset' + AND trigger_ref = $2 + AND runnable_kind = 'script' + "#, + workspace_id, + trigger_ref, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| Subscriber { + path: r.runnable_path, + join_all: r.join_all, + debounce_s: r.debounce_s, + retry_count: r.retry_count, + retry_delay_s: r.retry_delay_s, + }) + .collect()) +} + +async fn push_subscriber( + db: &DB, + producer: &MiniCompletedJob, + subscriber_path: &str, + asset_kind: AssetKind, + asset_path: &str, + producer_path: &str, + chain: &[String], + partition: Option<&str>, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +) -> Result { + // Same resolution as every other trigger path (`script_path_to_payload`): + // latest deployed hash plus the script's own runnable settings + // (concurrency, debounce, timeout), resolved through the + // runnable-settings handle. The cascade must not bypass a subscriber's + // concurrency limit just because it was triggered by an asset write. + let script = get_latest_deployed_hash_for_path( + None, + db.clone(), + &producer.workspace_id, + subscriber_path, + ) + .await? + .prefetch_cached(db) + .await?; + let hash = ScriptHash(script.hash); + let tag = script.tag; + let concurrency_settings = script.runnable_settings.concurrency_settings; + + // Debounce / retry semantics are a `private` feature (see `cascade`). + // OSS degrades both: debounce falls back to the subscriber's own + // script-level settings, retry is never applied. + let debouncing_settings = crate::cascade::cascade_debouncing_settings( + subscriber_path, + partition, + debounce_s, + script.runnable_settings.debouncing_settings, + ); + + // Retry is only available via the flow runtime — wrap the script in a + // one-step flow when the cascade declares one. No retry = + // unwrapped `ScriptHash` push. + let payload = if let Some(retry) = crate::cascade::cascade_retry(retry_count, retry_delay_s) { + JobPayload::SingleStepFlow { + path: subscriber_path.to_string(), + hash: Some(hash), + flow_version: None, + args: HashMap::new(), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + priority: script.priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor: false, + concurrency_settings, + debouncing_settings, + } + } else { + JobPayload::ScriptHash { + hash, + path: subscriber_path.to_string(), + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + dedicated_worker: script.dedicated_worker, + language: script.language, + priority: script.priority, + apply_preprocessor: false, + debouncing_settings, + concurrency_settings, + labels: script.labels, + } + }; + + // Run the subscriber under its deployer's identity — never the + // producer's. Subscriptions are workspace-wide, so attributing the run + // to the producer would let anyone who can deploy a `// on` script + // execute code with the permissions of whoever happens to write the + // asset (e.g. an admin's scheduled job). `on_behalf_of_email` (an + // explicit service-account opt-in at deploy) takes precedence for the + // email; otherwise the deployer's email is resolved from their + // username. + let permissioned_as = username_to_permissioned_as(&script.created_by); + let email = match script.on_behalf_of_email { + Some(obo) => obo, + None => { + get_email_from_permissioned_as(&permissioned_as, &producer.workspace_id, db).await? + } + }; + + let mut args: HashMap> = HashMap::new(); + let trigger_payload = serde_json::json!({ + "kind": "asset", + "asset_kind": serde_json::to_value(&asset_kind).expect("AssetKind serializes"), + "asset_path": asset_path, + "producer_path": producer_path, + "producer_job_id": producer.id.to_string(), + CHAIN_KEY: chain, + PARTITION_ARG: partition, + }); + args.insert(TRIGGER_ARG.to_string(), to_raw_value(&trigger_payload)); + // Carry the producer's resolved partition forward as a top-level arg so + // the subscriber's body can read it and the next cascade hop's + // `read_partition` picks it up — keeps the whole chain on one partition, + // resolved once at the top. Omitted entirely for non-partitioned chains. + if let Some(p) = partition { + args.insert(PARTITION_ARG.to_string(), to_raw_value(&p)); + } + + // Attribute the dispatched run to a synthetic user so audit logs reflect + // it came from the asset cascade, not the original human runner. + let pseudo_user = format!("asset-{producer_path}"); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (id, tx) = push( + db, + tx, + &producer.workspace_id, + payload, + PushArgs { args: &args, extra: None }, + &pseudo_user, + &email, + permissioned_as, + Some(producer_path), + None, + Some(producer_path.to_string()), + None, + None, + None, + None, + false, + false, + None, + true, + tag, + script.timeout, + None, + None, + None, + false, + None, + Some(TriggerMetadata::new( + Some(producer_path.to_string()), + JobTriggerKind::Asset, + )), + None, + ) + .await + .map_err(|e| error::Error::internal_err(format!("push asset-triggered job: {e:#}")))?; + tx.commit().await?; + Ok(id) +} diff --git a/backend/windmill-queue/src/cascade_oss.rs b/backend/windmill-queue/src/cascade_oss.rs new file mode 100644 index 0000000000..be6f93d732 --- /dev/null +++ b/backend/windmill-queue/src/cascade_oss.rs @@ -0,0 +1,60 @@ +//! OSS fallback for the asset-trigger cascade's AND-join / debounce / retry. +//! +//! The richer cascade semantics are a `private` feature (see `cascade_ee`). +//! In the public build they are absent and the cascade degrades to a plain +//! OR fan-out: every join always fires immediately, no slots are recorded or +//! reaped, debounce falls back to the subscriber's own script-level settings, +//! and retry is never applied (the subscriber pushes as a bare `ScriptHash`). +//! The real implementation lives in `windmill-ee-private`. + +use windmill_common::error::Result; +use windmill_common::flows::Retry; +use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::DB; + +/// Outcome of evaluating an AND-join barrier for one (subscriber, input). +/// Kept identical to the EE definition so the core matcher in +/// `asset_dispatch` compiles against either build. +pub enum JoinDecision { + /// Input does not advance the join (recorded as Skipped with `reason`). + Skip(&'static str), + /// Join advanced but is not yet complete (recorded as JoinPending). + Pending { received: i32, required: i32 }, + /// All required inputs are present — push the subscriber. + Fire, +} + +/// OSS has no AND-join: every input fires immediately (plain OR fan-out). +pub async fn handle_join( + _db: &DB, + _workspace_id: &str, + _sub_path: &str, + _trigger_ref: &str, + _partition: Option<&str>, +) -> Result { + Ok(JoinDecision::Fire) +} + +/// Unused in OSS (no slots are ever recorded), kept for API parity. +pub const JOIN_SLOT_TTL_SECS: i64 = 60 * 24 * 60 * 60; // 60 days + +/// No-op: OSS never records join slots, so there is nothing to reap. +pub async fn reap_stale_join_slots(_db: &DB) -> Result<()> { + Ok(()) +} + +/// No per-edge debounce in OSS — always defer to the subscriber's own +/// script-level debounce settings. +pub fn cascade_debouncing_settings( + _subscriber_path: &str, + _partition: Option<&str>, + _debounce_s: Option, + fallback: DebouncingSettings, +) -> DebouncingSettings { + fallback +} + +/// No per-edge retry in OSS — the subscriber pushes as a bare `ScriptHash`. +pub fn cascade_retry(_retry_count: Option, _retry_delay_s: Option) -> Option { + None +} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index be0d1590b6..6f689c7026 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -6,6 +6,14 @@ * LICENSE-AGPL for a copy of the license. */ +pub mod asset_dispatch; +#[cfg(feature = "private")] +pub mod cascade_ee; +pub mod cascade_oss; +#[cfg(feature = "private")] +pub use cascade_ee as cascade; +#[cfg(not(feature = "private"))] +pub use cascade_oss as cascade; pub mod jobs; #[cfg(feature = "private")] pub mod jobs_ee; diff --git a/backend/windmill-trigger-postgres/src/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs index 22612db8d1..14d348b5f2 100644 --- a/backend/windmill-trigger-postgres/src/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -7,10 +7,12 @@ use pg_escape::{quote_identifier, quote_literal}; use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; use tokio::sync::RwLock; use tokio_stream::StreamExt; +use windmill_api_auth::ApiAuthed; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, jobs::JobTriggerKind, + utils::{report_critical_error, report_recovered_critical_error}, worker::to_raw_value, DB, }; @@ -31,6 +33,10 @@ use super::{ const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; +// Wait this long between reconnection attempts after a connection failure or a +// dropped replication stream. Matches the Kafka trigger listener's backoff. +const RECONNECT_DELAY_SECS: u64 = 30; + pub struct LogicalReplicationSettings { pub streaming: bool, } @@ -126,9 +132,70 @@ impl PostgresSimpleClient { } } +/// Resolves the Postgres resource, validates that the configured publication and +/// replication slot still exist, and opens a fresh logical replication stream. +/// +/// Returns `Error::BadConfig` when the publication or slot is missing (an +/// unrecoverable misconfiguration). Any other error is treated as transient +/// (connection refused, network interruption, ...) and is retried by the caller. +/// The resource is re-resolved on every call so credential rotations are picked +/// up across reconnections. +async fn connect_logical_replication_stream( + authed: &ApiAuthed, + db: &DB, + listening_trigger: &ListeningTrigger, +) -> Result<(CopyBothDuplex, LogicalReplicationSettings)> { + let ListeningTrigger { workspace_id, trigger_config, .. } = listening_trigger; + let PostgresConfig { postgres_resource_path, publication_name, replication_slot_name, .. } = + trigger_config; + + let database = resolve_postgres_resource( + authed, + Some(UserDB::new(db.clone())), + db, + postgres_resource_path, + workspace_id, + ) + .await?; + + let client = PostgresSimpleClient::new(&database).await?; + + let publication = client + .execute_query(&format!( + "SELECT pubname FROM pg_publication WHERE pubname = {}", + quote_literal(publication_name) + )) + .await + .map_err(to_anyhow)?; + + if !publication.row_exist() { + return Err(Error::BadConfig( + ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), + )); + } + + let replication_slot = client + .execute_query(&format!( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", + quote_literal(replication_slot_name) + )) + .await + .map_err(to_anyhow)?; + + if !replication_slot.row_exist() { + return Err(Error::BadConfig( + ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), + )); + } + + client + .get_logical_replication_stream(publication_name, replication_slot_name) + .await +} + #[async_trait::async_trait] impl Listener for PostgresTrigger { - type Consumer = (CopyBothDuplex, LogicalReplicationSettings); + type Consumer = ApiAuthed; type Extra = (); type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; @@ -140,65 +207,13 @@ impl Listener for PostgresTrigger { _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> Result> { - let ListeningTrigger:: { workspace_id, trigger_config, .. } = - listening_trigger; - - let PostgresConfig { - postgres_resource_path, publication_name, replication_slot_name, .. - } = trigger_config; - + // The actual replication connection is established (and retried) inside + // `consume`. Here we only resolve the auth context that connection needs. let authed = listening_trigger .authed(db, &Self::TRIGGER_KIND.to_string()) .await?; - let database = resolve_postgres_resource( - &authed, - Some(UserDB::new(db.clone())), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - let client = PostgresSimpleClient::new(&database).await?; - - let publication = client - .execute_query(&format!( - "SELECT pubname FROM pg_publication WHERE pubname = {}", - quote_literal(&publication_name) - )) - .await - .map_err(to_anyhow)?; - - if !publication.row_exist() { - return Err(Error::BadConfig( - ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - )); - } - - let replication_slot = client - .execute_query(&format!( - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", - quote_literal(&replication_slot_name) - )) - .await - .map_err(to_anyhow)?; - - if !replication_slot.row_exist() { - return Err(Error::BadConfig( - ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - )); - } - - let (logical_replication_stream, logical_replication_settings) = client - .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await - .map_err(to_anyhow)?; - - Ok(Some(( - logical_replication_stream, - logical_replication_settings, - ))) + Ok(Some(authed)) } async fn consume( &self, @@ -209,218 +224,376 @@ impl Listener for PostgresTrigger { _killpill_rx: tokio::sync::broadcast::Receiver<()>, _extra_state: Option<&Self::ExtraState>, ) { - let (logical_replication_stream, logical_replication_settings) = consumer; - pin_mut!(logical_replication_stream); - let mut relations = RelationConverter::new(); - tracing::info!( - "Starting to listen for postgres trigger {}", - &listening_trigger.path - ); - - // Highest WAL position received (and processed) so far. Reported back - // to Postgres via standby status updates so the replication slot can - // advance and retained WAL gets released. Without periodic updates the - // slot's LSNs stay frozen and WAL grows unbounded. - let mut last_lsn: u64 = 0; - let mut status_interval = tokio::time::interval(Duration::from_secs(10)); - status_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // First tick resolves immediately; consume it so the periodic cadence - // starts one full interval from now. - status_interval.tick().await; + let authed = consumer; + // Consecutive failed connection attempts. Reset to 0 once the replication + // stream is (re)established. + let mut tries = 0_usize; loop { - let next = tokio::select! { - _ = status_interval.tick() => None, - message = logical_replication_stream.next() => Some(message), - }; - - let message = match next { - None => { - PostgresSimpleClient::send_status_update( - last_lsn, - &mut logical_replication_stream, - ) - .await; - continue; - } - Some(message) => message, - }; - let message = match message { - Some(message) => message, - None => { - tracing::error!( - "Stream for postgres trigger {} closed", - &listening_trigger.path - ); - if let None = self - .update_ping_and_loop_ping_status( - db, - listening_trigger, - err_message.clone(), - Some("Stream closed".to_string()), - ) - .await - { + let (logical_replication_stream, logical_replication_settings) = + match connect_logical_replication_stream(&authed, db, listening_trigger).await { + Ok(stream) => stream, + // Publication or replication slot missing: retrying cannot fix + // this, so disable the trigger as before. + Err(Error::BadConfig(err)) => { + self.disable_with_error(db, listening_trigger, err).await; return; } - return; - } - }; + // Transient failure (connection refused, network drop, ...): + // back off and retry instead of permanently disabling. + Err(err) => { + let status = format!( + "Failed to connect (attempt {}), retrying in {} seconds: {}", + tries + 1, + RECONNECT_DELAY_SECS, + err + ); + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some(status), + ) + .await + { + return; + } - let message = match message { - Ok(message) => message, - Err(err) => { - let err = format!( - "Postgres trigger named {} had an error while receiving a message : {}", - &listening_trigger.path, - err.to_string() - ); - self.disable_with_error(db, listening_trigger, err).await; - return; - } - }; + tracing::error!( + "Failed to connect postgres trigger {} (attempt {}), retrying in {} seconds: {}", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS, + err + ); - let logical_message = match ReplicationMessage::parse(message) { - Ok(logical_message) => logical_message, - Err(err) => { - let err = format!( - "Postgres trigger named: {} had an error while parsing message: {}", - &listening_trigger.path, - err.to_string() - ); - self.disable_with_error(db, listening_trigger, err).await; - return; - } - }; + if tries % 10 == 0 && listening_trigger.trigger_mode { + report_critical_error( + format!( + "Failed to connect postgres trigger {} (attempt {}), retrying in {} seconds. This alert will repeat every 10 failed attempts. Error: {}", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + Some(&format!("postgres_trigger:{}", &listening_trigger.path)), + ) + .await; + } - match logical_message { - ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { - last_lsn = last_lsn.max(primary_keep_alive.wal_end); - if primary_keep_alive.reply { + tries += 1; + tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECS)).await; + continue; + } + }; + + // The retry counter is reset (and recovery reported, ping cleared) only + // once the stream actually delivers a message in the inner loop. A + // connection that drops before making any progress therefore keeps + // counting toward the reconnection alert instead of ping-ponging + // silently. + pin_mut!(logical_replication_stream); + let mut relations = RelationConverter::new(); + tracing::info!( + "Starting to listen for postgres trigger {}", + &listening_trigger.path + ); + + // Highest WAL position received (and processed) so far. Reported back + // to Postgres via standby status updates so the replication slot can + // advance and retained WAL gets released. Without periodic updates the + // slot's LSNs stay frozen and WAL grows unbounded. + let mut last_lsn: u64 = 0; + let mut status_interval = tokio::time::interval(Duration::from_secs(10)); + status_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // First tick resolves immediately; consume it so the periodic cadence + // starts one full interval from now. + status_interval.tick().await; + + 'stream: loop { + let next = tokio::select! { + _ = status_interval.tick() => None, + message = logical_replication_stream.next() => Some(message), + }; + + let message = match next { + None => { PostgresSimpleClient::send_status_update( last_lsn, &mut logical_replication_stream, ) .await; + continue; } - } - ReplicationMessage::XLogData(x_log_data) => { - last_lsn = last_lsn.max(x_log_data.wal_end); - let logical_replication_message = match x_log_data - .parse(&logical_replication_settings) - { - Ok(logical_replication_message) => logical_replication_message, - Err(err) => { - tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); - continue; + Some(message) => message, + }; + let message = match message { + Some(message) => message, + None => { + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some(format!( + "Stream closed (attempt {}), reconnecting in {} seconds", + tries + 1, + RECONNECT_DELAY_SECS + )), + ) + .await + { + return; } - }; - - let json = match logical_replication_message { - Relation(relation_body) => { - relations.add_relation(relation_body); - None - } - Begin | Type | Commit => None, - Insert(insert) => Some(( - insert.o_id, - Ok(None), - relations.row_to_json((insert.o_id, insert.tuple)), - "insert", - )), - Update(update) => { - let old_row = update - .old_tuple - .map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))) - .transpose(); - let row = relations.row_to_json((update.o_id, update.new_tuple)); - Some((update.o_id, old_row, row, "update")) - } - Delete(delete) => { - let row = delete - .old_tuple - .unwrap_or_else(|| delete.key_tuple.unwrap()); - Some(( - delete.o_id, - Ok(None), - relations.row_to_json((delete.o_id, row)), - "delete", - )) - } - }; - match json { - Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!( - "Postgres trigger named: {}, error: {}", - &listening_trigger.path, - err.to_string() - ); - continue; - } - }; - let database_info = HashMap::from([ - ("schema_name".to_string(), to_raw_value(&relation.namespace)), - ("table_name".to_string(), to_raw_value(&relation.name)), - ( - "transaction_type".to_string(), - to_raw_value(&transaction_type), + tracing::error!( + "Stream for postgres trigger {} closed (attempt {}), reconnecting in {} seconds", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS + ); + if tries % 10 == 0 && listening_trigger.trigger_mode { + report_critical_error( + format!( + "Postgres trigger {} stream closed (attempt {}), reconnecting in {} seconds. This alert will repeat every 10 failed attempts.", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS ), - ("old_row".to_string(), to_raw_value(&old_row)), - ("row".to_string(), to_raw_value(&row)), - ]); - let _ = self - .handle_event( - db, - listening_trigger, - database_info, - HashMap::new(), - None, - ) - .await; + db.clone(), + Some(&listening_trigger.workspace_id), + Some(&format!("postgres_trigger:{}", &listening_trigger.path)), + ) + .await; } - Some((o_id, old_row, row, transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { + tries += 1; + tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECS)).await; + break 'stream; + } + }; + + let message = match message { + Ok(message) => message, + Err(err) => { + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some(format!( + "Error receiving message (attempt {}), reconnecting in {} seconds: {}", + tries + 1, + RECONNECT_DELAY_SECS, + err + )), + ) + .await + { + return; + } + tracing::error!( + "Postgres trigger {} had an error while receiving a message (attempt {}), reconnecting in {} seconds: {}", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS, + err.to_string() + ); + if tries % 10 == 0 && listening_trigger.trigger_mode { + report_critical_error( + format!( + "Postgres trigger {} error while receiving a message (attempt {}), reconnecting in {} seconds. This alert will repeat every 10 failed attempts. Error: {}", + &listening_trigger.path, + tries + 1, + RECONNECT_DELAY_SECS, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + Some(&format!("postgres_trigger:{}", &listening_trigger.path)), + ) + .await; + } + tries += 1; + tokio::time::sleep(Duration::from_secs(RECONNECT_DELAY_SECS)).await; + break 'stream; + } + }; + + // First successful read after a (re)connection means the stream is + // making progress: clear the error status, report recovery, and + // reset the retry counter. Deferred to here (rather than on connect) + // so a stream that drops before delivering anything keeps counting + // toward the reconnection alert. + if tries > 0 { + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + None, + ) + .await + { + return; + } + if listening_trigger.trigger_mode { + report_recovered_critical_error( + format!("Postgres trigger {} reconnected", &listening_trigger.path), + db.clone(), + Some(&listening_trigger.workspace_id), + Some(&format!("postgres_trigger:{}", &listening_trigger.path)), + ) + .await; + } + tries = 0; + } + + let logical_message = match ReplicationMessage::parse(message) { + Ok(logical_message) => logical_message, + Err(err) => { + let err = format!( + "Postgres trigger named: {} had an error while parsing message: {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + match logical_message { + ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { + last_lsn = last_lsn.max(primary_keep_alive.wal_end); + if primary_keep_alive.reply { + PostgresSimpleClient::send_status_update( + last_lsn, + &mut logical_replication_stream, + ) + .await; + } + } + ReplicationMessage::XLogData(x_log_data) => { + last_lsn = last_lsn.max(x_log_data.wal_end); + let logical_replication_message = match x_log_data + .parse(&logical_replication_settings) + { + Ok(logical_replication_message) => logical_replication_message, + Err(err) => { + tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); + continue; + } + }; + + let json = match logical_replication_message { + Relation(relation_body) => { + relations.add_relation(relation_body); + None + } + Begin | Type | Commit => None, + Insert(insert) => Some(( + insert.o_id, + Ok(None), + relations.row_to_json((insert.o_id, insert.tuple)), + "insert", + )), + Update(update) => { + let old_row = update + .old_tuple + .map(|old_tuple| { + relations.row_to_json((update.o_id, old_tuple)) + }) + .transpose(); + let row = relations.row_to_json((update.o_id, update.new_tuple)); + Some((update.o_id, old_row, row, "update")) + } + Delete(delete) => { + let row = delete + .old_tuple + .unwrap_or_else(|| delete.key_tuple.unwrap()); + Some(( + delete.o_id, + Ok(None), + relations.row_to_json((delete.o_id, row)), + "delete", + )) + } + }; + match json { + Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + let database_info = HashMap::from([ + ("schema_name".to_string(), to_raw_value(&relation.namespace)), + ("table_name".to_string(), to_raw_value(&relation.name)), + ( + "transaction_type".to_string(), + to_raw_value(&transaction_type), + ), + ("old_row".to_string(), to_raw_value(&old_row)), + ("row".to_string(), to_raw_value(&row)), + ]); + let _ = self + .handle_event( + db, + listening_trigger, + database_info, + HashMap::new(), + None, + ) + .await; + } + Some((o_id, old_row, row, transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + + if let Err(err) = old_row { tracing::error!( - "Postgres trigger named: {}, error: {}", - &listening_trigger.path, - err.to_string() + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode OLD row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, ); - continue; } - }; - if let Err(err) = old_row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode OLD row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - if let Err(err) = row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode NEW row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); + if let Err(err) = row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode NEW row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } } + _ => {} } - _ => {} } } } diff --git a/backend/windmill-trigger-websocket/src/handler.rs b/backend/windmill-trigger-websocket/src/handler.rs index dd6b9f4363..ed8e95b164 100644 --- a/backend/windmill-trigger-websocket/src/handler.rs +++ b/backend/windmill-trigger-websocket/src/handler.rs @@ -15,8 +15,8 @@ use windmill_git_sync::DeployedObject; use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ - get_url_from_runnable_value, proxy::connect_async_with_proxy, TestWebsocketConfig, - WebsocketConfig, WebsocketConfigRequest, WebsocketTrigger, + get_url_from_runnable_value, proxy::connect_async_with_proxy, validate_websocket_url_for_ssrf, + TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, WebsocketTrigger, }; #[async_trait] @@ -28,7 +28,8 @@ impl TriggerCrud for WebsocketTrigger { const TABLE_NAME: &'static str = "websocket_trigger"; const TRIGGER_TYPE: &'static str = "websocket"; - const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = windmill_common::user_drafts::UserDraftItemKind::TriggerWebsocket; + const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = + windmill_common::user_drafts::UserDraftItemKind::TriggerWebsocket; const SUPPORTS_SERVER_STATE: bool = true; const SUPPORTS_TEST_CONNECTION: bool = true; const ROUTE_PREFIX: &'static str = "/websocket_triggers"; @@ -61,6 +62,13 @@ impl TriggerCrud for WebsocketTrigger { )); } + // Reject SSRF targets at save time for static URLs. A `$flow:`/`$script:` + // URL is only known at runtime, so it is validated at connect time + // instead (in the listener and test handler). + if !config.url.starts_with('$') { + validate_websocket_url_for_ssrf(&config.url).await?; + } + if let Some(args) = &config.url_runnable_args { if !args.is_object() { return Err(Error::BadRequest( @@ -277,6 +285,8 @@ impl TriggerCrud for WebsocketTrigger { Cow::Borrowed(&url) }; + validate_websocket_url_for_ssrf(&connect_url).await?; + connect_async_with_proxy(&*connect_url) .await .map_err(|err| { diff --git a/backend/windmill-trigger-websocket/src/lib.rs b/backend/windmill-trigger-websocket/src/lib.rs index e61c067479..4111606b2b 100644 --- a/backend/windmill-trigger-websocket/src/lib.rs +++ b/backend/windmill-trigger-websocket/src/lib.rs @@ -104,6 +104,58 @@ pub fn value_to_args_hashmap( Ok(args) } +/// Env var that opts a deployment out of SSRF validation for WebSocket trigger +/// URLs, permitting connections to private/internal addresses. Off by default. +pub const ALLOW_PRIVATE_WEBSOCKET_URLS_ENV: &str = "ALLOW_PRIVATE_WEBSOCKET_URLS"; + +/// Reject WebSocket URLs that target (or resolve to) a private/internal address, +/// blocking SSRF probes of the host's internal network and cloud metadata +/// endpoints. +/// +/// `ws://`/`wss://` are mapped to `http`/`https` so the shared +/// `validate_url_for_ssrf` host + DNS-resolution checks apply. The +/// security-critical call sites are the outbound connects (the test handler and +/// every listener (re)connect): validating the *resolved* URL there means a +/// `$flow:`/`$script:` URL is checked on its returned value and re-checked on +/// each reconnect (DNS rebinding). `validate_config` also calls this at save +/// time to reject static URLs early. +pub async fn validate_websocket_url_for_ssrf(url: &str) -> Result<()> { + if std::env::var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") + { + return Ok(()); + } + + // `ws`/`wss` aren't recognised by `validate_url_for_ssrf`'s scheme check, so + // map them to the http(s) equivalent the same connection would tunnel over. + // The prefixes are ASCII, so byte-slicing at their length stays on a char + // boundary. + let lower = url.to_ascii_lowercase(); + let http_url = if lower.starts_with("wss://") { + format!("https://{}", &url["wss://".len()..]) + } else if lower.starts_with("ws://") { + format!("http://{}", &url["ws://".len()..]) + } else { + url.to_string() + }; + + windmill_common::ssrf::validate_url_for_ssrf(&http_url) + .await + .map_err(|e| match e { + // The env-var hint is only actionable for a well-formed URL blocked + // for targeting a private address; a malformed URL or bad scheme + // surfaces its real error so the user fixes the URL (see #9171). + e @ windmill_common::ssrf::SsrfValidationError::Private { .. } => { + Error::BadRequest(format!( + "{e}. If you need to connect to private/internal WebSocket endpoints, \ + set the {ALLOW_PRIVATE_WEBSOCKET_URLS_ENV}=true environment variable" + )) + } + e => Error::from(e), + }) +} + pub async fn get_url_from_runnable_value( path: &str, is_flow: bool, @@ -144,3 +196,40 @@ pub async fn get_url_from_runnable_value( )) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn ssrf_blocks_private_and_metadata_ws_urls() { + // ws:// → http:// mapping must still reach the IP-literal block. + let err = validate_websocket_url_for_ssrf("ws://127.0.0.1:6379/") + .await + .unwrap_err(); + assert!(matches!(err, Error::BadRequest(_))); + // Private errors carry the opt-out hint so operators can allow internal + // targets deliberately. + assert!(err.to_string().contains(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV)); + + // wss:// → https:// mapping blocks the cloud metadata endpoint. + assert!( + validate_websocket_url_for_ssrf("wss://169.254.169.254/latest/meta-data") + .await + .is_err() + ); + assert!(validate_websocket_url_for_ssrf("ws://10.0.0.5:6379/") + .await + .is_err()); + } + + #[tokio::test] + async fn ssrf_rejects_non_ws_scheme_without_private_hint() { + // A non-ws scheme isn't mapped and fails the scheme check; it must not + // get the "set ALLOW_PRIVATE_WEBSOCKET_URLS" hint (issue #9171). + let err = validate_websocket_url_for_ssrf("file:///etc/passwd") + .await + .unwrap_err(); + assert!(!err.to_string().contains(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV)); + } +} diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index 54dd4ddf60..31e3f83995 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -1,5 +1,6 @@ use super::{ - get_url_from_runnable_value, proxy::connect_async_with_proxy, WebsocketConfig, WebsocketTrigger, + get_url_from_runnable_value, proxy::connect_async_with_proxy, validate_websocket_url_for_ssrf, + WebsocketConfig, WebsocketTrigger, }; use anyhow::Context; use async_trait::async_trait; @@ -173,6 +174,8 @@ impl Listener for WebsocketTrigger { Cow::Borrowed(&url) }; + validate_websocket_url_for_ssrf(&connect_url).await?; + let connection = connect_async_with_proxy(&*connect_url) .await .map(|conn| Some(conn)) diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 4f3d381569..65072bc9ab 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -1057,6 +1057,10 @@ async fn test_connection( Path(workspace_id): Path, Json(config): Json, ) -> Result<()> { + // Test connection opens an outbound connection to a caller-supplied target, + // so gate it behind write access like the other mutating trigger routes. + check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; + let connect_f = async move { handler .test_connection(&db, &authed, &user_db, &workspace_id, config) diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index be20cfae3e..fb624382c4 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -16,6 +16,23 @@ pub enum AssetKind { Volume, } +impl AssetKind { + /// The canonical URI prefix used in asset trigger refs (e.g. `s3://`, + /// `$res:`). Single source of truth for both trigger-ref construction + /// and runtime cascade dispatch. `Variable` is deprecated and has no + /// canonical ref, so it returns `None`. + pub fn canonical_prefix(&self) -> Option<&'static str> { + match self { + AssetKind::S3Object => Some("s3://"), + AssetKind::Resource => Some("$res:"), + AssetKind::Ducklake => Some("ducklake://"), + AssetKind::DataTable => Some("datatable://"), + AssetKind::Volume => Some("volume://"), + AssetKind::Variable => None, + } + } +} + #[derive( Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, )] diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 75cde586e9..ac15430759 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -43,6 +43,9 @@ pub enum JobTriggerKind { #[serde(rename = "ci_test")] #[sqlx(rename = "ci_test")] CiTest, + // A run dispatched because an upstream pipeline script wrote an asset + // this runnable subscribes to via `// on s3://...` annotations. + Asset, } impl std::fmt::Display for JobTriggerKind { @@ -64,6 +67,7 @@ impl std::fmt::Display for JobTriggerKind { JobTriggerKind::Google => "google", JobTriggerKind::Github => "github", JobTriggerKind::CiTest => "ci_test", + JobTriggerKind::Asset => "asset", }; write!(f, "{}", kind) } diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 13fc00ddf9..6b4cd8cfd8 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -2,7 +2,8 @@ INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" ) EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" ) -TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST") +TRUSTED_HOST_ARG="" +for h in $TRUSTED_HOST; do TRUSTED_HOST_ARG="$TRUSTED_HOST_ARG --trusted-host $h"; done if [ ! -z "$INDEX_URL" ] then diff --git a/backend/windmill-worker/src/agent_workers.rs b/backend/windmill-worker/src/agent_workers.rs index 9bf0ea1834..5320c8ef6c 100644 --- a/backend/windmill-worker/src/agent_workers.rs +++ b/backend/windmill-worker/src/agent_workers.rs @@ -78,4 +78,22 @@ pub async fn get_datatable_resource_from_agent_http( .await } +/// Record a materialization outcome from an agent worker (no direct DB) via the +/// API, so `materialized_partition` state lands the same as on a Sql worker. +// Only called from the duckdb executor, which is itself `#[cfg(feature = "duckdb")]`. +#[cfg(feature = "duckdb")] +pub async fn record_materialization_from_agent_http( + client: &HttpClient, + w_id: &str, + req: &windmill_common::materialization::RecordMaterializationRequest, +) -> anyhow::Result<()> { + client + .post( + &format!("/api/w/{}/agent_workers/record_materialization", w_id), + None, + req, + ) + .await +} + pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping"; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 7b0ae3b3ea..dec0623e36 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -22,7 +22,8 @@ use windmill_common::{ use windmill_queue::MiniPulledJob; use windmill_parser_yaml::{ - AnsibleRequirements, GitRepo, PreexistingAnsibleInventory, ResourceOrVariablePath, + validate_vault_id, AnsibleRequirements, GitRepo, PreexistingAnsibleInventory, + ResourceOrVariablePath, }; use windmill_queue::{append_logs, CanceledBy}; @@ -910,6 +911,11 @@ pub fn create_ansible_cfg( } if let Some(vault_ids) = reqs.as_ref().map(|r| &r.vault_id) { if !vault_ids.is_empty() { + // Defense in depth: entries are validated at parse time, but re-check here + // since they are interpolated raw into ansible.cfg (config-directive injection). + for vault_id in vault_ids { + validate_vault_id(vault_id)?; + } let password_files = vault_ids.join(","); passwords_cfg.push_str(&format!("vault_identity_list = {password_files}\n")); @@ -1799,4 +1805,31 @@ mod tests { assert!(validate_relative_path("", "playbook").is_err()); assert!(validate_relative_path(" ", "playbook").is_err()); } + + #[test] + fn test_create_ansible_cfg_writes_valid_vault_id() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let reqs = AnsibleRequirements { + vault_id: vec!["dev@vault_pass.txt".to_string()], + ..Default::default() + }; + create_ansible_cfg(Some(&reqs), job_dir, false).unwrap(); + let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); + assert!(cfg.contains("vault_identity_list = dev@vault_pass.txt")); + assert!(!cfg.contains("library")); + } + + #[test] + fn test_create_ansible_cfg_rejects_vault_id_injection() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let reqs = AnsibleRequirements { + vault_id: vec!["default@/tmp/wm/x\nlibrary = /tmp/wm/evil_modules".to_string()], + ..Default::default() + }; + // Defense-in-depth boundary: a poisoned entry must error before any config is written. + assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err()); + assert!(!dir.path().join("ansible.cfg").exists()); + } } diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 1a474ee9a0..3b6a07f702 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -32,6 +32,178 @@ use crate::sql_utils::remove_comments; use windmill_common::client::AuthedClient; use windmill_object_store::DEFAULT_STORAGE; +// What a `// materialize` run records into `materialized_partition` once it +// finishes. `asset_path` is the full `/
` (the asset identity); +// `partition` is "" for an unpartitioned (whole-table) materialization. +struct MaterializeExec { + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + partition: String, +} + +// If `query` declares `// materialize `, return what to record plus, +// for the default managed mode, the rewritten managed-write SQL (in `manual` +// mode the script writes its own DDL, so the rewrite is `None`). The rewritten +// SQL contains a synthetic `ATTACH 'ducklake://' AS _wm_target` that the +// normal ATTACH-transform pass resolves to real credentials — the same path as +// the user's own ATTACH. Returns `None` when there is no materialize annotation +// or the target isn't a ducklake (only ducklake is materialized in v1). +fn build_materialized_query( + query: &str, + partition_value: Option<&str>, +) -> Result, MaterializeExec)>> { + use windmill_parser::asset_parser::{parse_pipeline_annotations, AssetKind as PAssetKind}; + use windmill_parser::sql_materialize::{ + build_wrap_blocks, classify_wrap, MaterializeStrategy, TARGET_ALIAS, + }; + + let ann = parse_pipeline_annotations(query); + let Some(m) = ann.materialize else { + return Ok(None); + }; + if m.target_kind != PAssetKind::Ducklake { + return Ok(None); + } + let partitioned = ann.partition.is_some(); + let partition = partition_value.unwrap_or("").to_string(); + // Partition *resolution* is enterprise; in its absence a partitioned + // materialize only runs with an explicit `partition` arg. Fail loudly rather + // than silently materialize the wrong (empty) slice. + if partitioned && partition.is_empty() { + return Err(Error::ExecutionErr( + "materialize: a `// partitioned` script ran with no resolved partition — pass an \ + explicit `partition` arg, or enable enterprise partition resolution" + .to_string(), + )); + } + // Convention: `ducklake:///
` — is the configured + // ducklake (resolved like a user ATTACH),
is the rest. + let (ducklake_name, table) = m + .target_path + .split_once('/') + .unwrap_or((m.target_path.as_str(), "")); + let meta = MaterializeExec { + asset_kind: windmill_common::assets::AssetKind::Ducklake, + asset_path: m.target_path.clone(), + partition: partition.clone(), + }; + + if m.manual { + // Escape hatch: the script owns its DDL; we only record state. + return Ok(Some((None, meta))); + } + if table.is_empty() { + return Err(Error::ExecutionErr(format!( + "materialize: target `ducklake://{}` has no table (use ducklake:///
)", + m.target_path + ))); + } + let mut plan = classify_wrap(query).map_err(|e| Error::ExecutionErr(e.message()))?; + // Resolve the `{partition}` token (same token `// on` asset URIs use) to the + // current partition value everywhere in the managed script, so a partitioned + // materialize can filter its source by the active slice, e.g. + // `WHERE day = {partition}`. The token is always replaced by a *complete* + // escaped SQL literal (`'…'` with `'` doubled) whether or not the author + // quoted it — so a run caller can't pass metacharacters that break out of + // the literal and alter statement boundaries. The pre-quoted form + // `'{partition}'` is matched first so it doesn't become `''…''`. Only + // meaningful when partitioned. + if partitioned { + let lit = format!("'{}'", partition.replace('\'', "''")); + let tok = windmill_common::assets::PARTITION_TOKEN; + let quoted_tok = format!("'{tok}'"); + plan.output = plan.output.replace("ed_tok, &lit).replace(tok, &lit); + for s in plan.setup.iter_mut() { + *s = s.replace("ed_tok, &lit).replace(tok, &lit); + } + } + let strategy = if m.append { + MaterializeStrategy::Append + } else if let Some(uk) = m.unique_key { + MaterializeStrategy::Merge { unique_key: uk } + } else { + MaterializeStrategy::Replace + }; + // Inline the partition as an escaped SQL literal (DuckLake has no bind for + // the partition column in our generated DDL). + let pval = format!("'{}'", partition.replace('\'', "''")); + let synthetic_attach = format!("ATTACH 'ducklake://{ducklake_name}' AS {TARGET_ALIAS};"); + let blocks = build_wrap_blocks( + &plan, + &synthetic_attach, + table, + &m.target_path, + "_wm_partition", + &pval, + partitioned, + strategy, + ); + Ok(Some((Some(blocks.join("\n")), meta))) +} + +// Pull a named i64 field (`snapshot_id` / `rows`) out of the trailing summary +// read — which in wrap mode is the job result. Shape-tolerant (object / array / +// nested), returns None if absent (literal mode, or capture failed). +fn extract_i64(result: &RawValue, field: &str) -> Option { + fn find(v: &Value, field: &str) -> Option { + match v { + Value::Number(n) => n.as_i64(), + Value::Object(m) => m.get(field).and_then(|x| find(x, field)), + Value::Array(a) => a.iter().find_map(|x| find(x, field)), + _ => None, + } + } + find(&serde_json::from_str::(result.get()).ok()?, field) +} + +// Best-effort record of a materialization outcome. On a Sql connection it writes +// the row directly; on an agent worker (Http, no direct DB) it posts to the API +// so state lands the same way. Never fails the job — a lost row degrades the +// grid, not the run. +async fn record_mat( + conn: &Connection, + w_id: &str, + job_id: Uuid, + meta: &MaterializeExec, + status: windmill_common::materialization::MaterializationStatus, + snapshot_id: Option, + row_count: Option, + error: Option<&str>, +) { + let req = windmill_common::materialization::RecordMaterializationRequest { + asset_kind: meta.asset_kind, + asset_path: meta.asset_path.clone(), + partition: meta.partition.clone(), + status, + snapshot_id, + row_count, + job_id: Some(job_id), + error: error.map(|e| e.to_string()), + }; + let res: anyhow::Result<()> = match conn { + Connection::Sql(db) => windmill_common::materialization::record_materialization( + db, + w_id, + req.asset_kind, + &req.asset_path, + &req.partition, + req.status, + req.snapshot_id, + req.row_count, + req.job_id, + req.error.as_deref(), + ) + .await + .map_err(|e| anyhow::anyhow!("{e:#}")), + Connection::Http(client) => { + crate::agent_workers::record_materialization_from_agent_http(client, w_id, &req).await + } + }; + if let Err(e) = res { + tracing::warn!("failed to record materialization state: {e:#}"); + } +} + pub async fn do_duckdb( job: &MiniPulledJob, client: &AuthedClient, @@ -68,6 +240,30 @@ pub async fn do_duckdb( let mut hidden_passwords = hidden_passwords.clone(); let mut bigquery_credentials = None; + // Materialization (`// materialize`): rewrite a wrap script into managed + // DDL (its synthetic target ATTACH is resolved by the transform pass + // below, like the user's own ATTACH); a literal script is left as-is. + // `materialize` also carries what to record once the run finishes. + let partition_value: Option = job + .args + .as_ref() + .and_then(|a| a.0.get(windmill_common::partition::PARTITION_ARG)) + .and_then(|rv| serde_json::from_str::(rv.get()).ok()) + .filter(|s| !s.is_empty()); + let materialize = if query.contains("materialize") { + build_materialized_query(query, partition_value.as_deref())? + } else { + None + }; + let materialized_query; + let query: &str = match &materialize { + Some((Some(rewritten), _)) => { + materialized_query = rewritten.clone(); + &materialized_query + } + _ => query, + }; + let sig = parse_duckdb_sig(query)?.args; let mut job_args = build_args_values(job, client, conn).await?; @@ -199,6 +395,19 @@ pub async fn do_duckdb( let (result, column_order) = match result { Ok(r) => r, Err(e) => { + if let Some((_, meta)) = &materialize { + record_mat( + conn, + &job.workspace_id, + job.id, + meta, + windmill_common::materialization::MaterializationStatus::Failed, + None, + None, + Some(&e.to_string()), + ) + .await; + } if let Some(s3_proxy_err) = S3_PROXY_LAST_ERRORS_CACHE.get(&client.token) { return Err(Error::ExecutionErr(format!( "{}\n\nS3 Related Error: {}", @@ -210,6 +419,24 @@ pub async fn do_duckdb( } }; + if let Some((_, meta)) = &materialize { + // In wrap mode the job result is the summary read (snapshot_id + + // rows); in literal mode there is none, so both stay None. + let snapshot_id = extract_i64(&result, "snapshot_id"); + let row_count = extract_i64(&result, "rows"); + record_mat( + conn, + &job.workspace_id, + job.id, + meta, + windmill_common::materialization::MaterializationStatus::Materialized, + snapshot_id, + row_count, + None, + ) + .await; + } + drop(bigquery_credentials); *column_order_ref = column_order; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index f74a523072..409dc362af 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -24,8 +24,10 @@ use tokio::{ use windmill_queue::MiniPulledJob; use uuid::Uuid; -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] + +#[cfg(all(feature = "enterprise", feature = "parquet"))] use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + use windmill_common::{ error::{ self, @@ -72,7 +74,7 @@ lazy_static::lazy_static! { static ref EPHEMERAL_TOKEN_CMD: Option = var("EPHEMERAL_TOKEN_CMD").ok(); } -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] +#[cfg(all(feature = "enterprise", feature = "parquet"))] lazy_static::lazy_static! { static ref PIPTAR_UPLOAD_CHANNEL: tokio::sync::mpsc::UnboundedSender = { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); @@ -84,14 +86,14 @@ lazy_static::lazy_static! { }; } -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] +#[cfg(all(feature = "enterprise", feature = "parquet"))] #[derive(Debug)] struct PiptarUploadTask { venv_path: String, cache_dir: String, } -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] +#[cfg(all(feature = "enterprise", feature = "parquet"))] async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver) { use crate::global_cache::build_tar_and_push; use windmill_object_store::get_object_store; @@ -137,10 +139,10 @@ pub fn has_relative_imports(content: &str) -> bool { RELATIVE_IMPORT_REGEX.is_match(content) } -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] +#[cfg(all(feature = "enterprise", feature = "parquet"))] use crate::global_cache::pull_from_tar; -#[cfg(all(feature = "enterprise", feature = "parquet", unix))] +#[cfg(all(feature = "enterprise", feature = "parquet"))] use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ @@ -361,7 +363,9 @@ pub async fn uv_pip_compile( args.extend(["--index-url", url]); } if let Some(host) = TRUSTED_HOST.as_ref() { - args.extend(["--trusted-host", host]); + host.split_whitespace().for_each(|h| { + args.extend(["--trusted-host", h]); + }); } if let Some(cert_path) = INDEX_CERT.as_ref() { args.extend(["--cert", cert_path]); @@ -2192,7 +2196,9 @@ async fn spawn_uv_install( command_args.extend(["--index-url", url]); } if let Some(host) = TRUSTED_HOST.as_ref() { - command_args.extend(["--trusted-host", &host]); + host.split_whitespace().for_each(|h| { + command_args.extend(["--trusted-host", h]); + }); } if *NATIVE_CERT { command_args.extend(["--native-tls"]); @@ -2384,12 +2390,12 @@ pub async fn handle_python_reqs( instant: std::time::Instant, conn: &Connection, ) { - #[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))] + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] { (s3_pull, s3_push) = (false, false); } - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] if OBJECT_STORE_SETTINGS.read().await.is_none() { (s3_pull, s3_push) = (false, false); } @@ -2644,7 +2650,7 @@ pub async fn handle_python_reqs( let mut handles = Vec::with_capacity(total_to_install); // let mem_peak_thread_safe = Arc::new(tokio::sync::Mutex::new(0)); - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro); let total_time = std::time::Instant::now(); @@ -2692,7 +2698,7 @@ pub async fn handle_python_reqs( let pids = pids.clone(); let worker_dir = worker_dir.clone(); - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] let py_version = py_version.clone(); handles.push(task::spawn(async move { @@ -2710,7 +2716,7 @@ pub async fn handle_python_reqs( ); let start = std::time::Instant::now(); - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] if is_not_pro { if let Some(os) = windmill_object_store::get_object_store().await { tokio::select! { @@ -2892,10 +2898,10 @@ pub async fn handle_python_reqs( } }; - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] let s3_push = is_not_pro; - #[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))] + #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let s3_push = false; if is_sandboxing_enabled() { @@ -2949,7 +2955,7 @@ pub async fn handle_python_reqs( ) .await; - #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + #[cfg(all(feature = "enterprise", feature = "parquet"))] if s3_push { // Send to upload channel for sequential processing let upload_task = PiptarUploadTask { diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 1824a9e493..ab2cad4da9 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -35,9 +35,9 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_mini_completed_job, is_pre_shaped_wm_failure_result, CanceledBy, FlowRunners, - JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG, - MANUAL_FAILURE_ERROR_NAME, + append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, + CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, + WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -779,13 +779,16 @@ pub async fn process_completed_job( )) })?; } else if let Some(preprocessed_args) = preprocessed_args { - // Update script args to preprocessed args - sqlx::query!( - "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", - Json(preprocessed_args) as Json>>, - job.id + // Update script args to preprocessed args, but preserve a + // resolved pipeline `partition` (injected before the body ran + // by resolve_partition_for_job). Run identity is immutable — + // the preprocessor must not change or drop it, or the asset + // cascade would read no partition for this producer. + windmill_common::partition::merge_args_preserving_partition( + db, + job.id, + preprocessed_args, ) - .execute(db) .await?; } @@ -805,6 +808,12 @@ pub async fn process_completed_job( from_cache.unwrap_or(false), ) .await?; + + // Asset-trigger fan-out: best-effort, never propagates errors. + // Internal eligibility checks gate to top-level Script/Preview runs; + // see windmill_queue::asset_dispatch. + asset_dispatch::dispatch_asset_triggers(db, &job).await; + drop(job); add_time!(bench, "add_completed_job END"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8964beb6be..45ae8c3c25 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3808,6 +3808,9 @@ pub async fn handle_queued_job( worker_name, flow_runners, &killpill_rx, + // A freshly pulled flow job is being executed by a live worker; the prior + // step (if any) completed normally, so this is never unrecoverable here. + false, )) .warn_after_seconds(10) .await @@ -4301,6 +4304,114 @@ async fn try_validate_schema( Ok(()) } +/// Pipeline partition resolution at execution time. The script content is +/// already loaded for this job, so parsing the `// partitioned` annotation +/// here is free (no extra fetch / no DB column). The concrete partition +/// value is resolved exactly once — schedule fire-time for time kinds +/// (anchored on `scheduled_for`, NOT wall-clock, so a chain crossing +/// midnight stays coherent) or the triggering payload for `dynamic`. It is +/// then (a) injected into the in-memory args the body sees and (b) +/// persisted back to `v2_job.args` so the asset-dispatch cascade reads the +/// same value at completion and propagates it downstream (run identity is +/// immutable — never re-resolve once set). +/// +/// `Ok(Some(job))` = a value was injected (caller must use the returned +/// clone). `Ok(None)` = nothing to do (no `// partitioned`, or the +/// partition is already set: explicit / backfill / cascade-propagated, or +/// before the `start` anchor). `Err` fails the job with a clear message +/// (partitioned but unresolvable — e.g. `dynamic` with no payload). +async fn resolve_partition_for_job( + job: &MiniPulledJob, + code: &str, + conn: &Connection, +) -> error::Result<(Option, bool)> { + use windmill_common::partition::{resolve_partition, PARTITION_ARG}; + use windmill_parser::asset_parser::PartitionKind; + + // Only deployed scripts participate in asset pipelines. Cheap substring + // guard so the overwhelming majority of script jobs skip the annotation + // scan; when one might be present we parse *once* here and reuse the result + // for both `in_pipeline` (→ WM_PIPELINE env, read by the wmll.ducklake SDK to + // record state) and `partition` resolution — no second parse downstream. The + // bool is whether the script is a `// pipeline` member. + if !matches!(job.kind, JobKind::Script) + || !(code.contains("pipeline") || code.contains("partitioned")) + { + return Ok((None, false)); + } + let ann = windmill_parser::asset_parser::parse_pipeline_annotations(code); + let in_pipeline = ann.in_pipeline; + let Some(spec) = ann.partition else { + return Ok((None, in_pipeline)); + }; + + // Already resolved upstream — explicit run arg, backfill, or + // cascade-propagated (push_subscriber injects a top-level `partition`). + // Run identity is immutable: use it as-is, do not re-resolve. + let already_set = job.args.as_ref().is_some_and(|a| { + a.0.get(PARTITION_ARG) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .is_some_and(|s| !s.is_empty()) + }); + if already_set { + return Ok((None, in_pipeline)); + } + + // `dynamic` extracts from the triggering payload (the `trigger` object + // for a cascade/event hop, else the run args themselves). Time kinds + // ignore the payload. + let payload: Option = match &spec.kind { + PartitionKind::Dynamic { .. } => job.args.as_ref().map(|a| { + a.0.get("trigger") + .and_then(|t| serde_json::from_str::(t.get()).ok()) + .unwrap_or_else(|| { + serde_json::Value::Object( + a.0.iter() + .filter_map(|(k, v)| { + serde_json::from_str(v.get()).ok().map(|jv| (k.clone(), jv)) + }) + .collect(), + ) + }) + }), + _ => None, + }; + + let resolved = resolve_partition(&spec, job.scheduled_for, payload.as_ref()) + .map_err(|e| Error::ExecutionErr(format!("partition resolution failed: {e:#}")))?; + let Some(value) = resolved else { + // Before the `start` anchor: this run has no partition to + // materialize. v1 runs it without one (logged) rather than + // introducing a skip-the-queued-job mechanism. + tracing::warn!( + job_id = %job.id, + "partitioned script resolved to no partition (before start anchor); running without one" + ); + return Ok((None, in_pipeline)); + }; + + // Persist back so dispatch_asset_triggers (which reads the producer's + // completed v2_job.args) propagates the same value down the cascade. + if let Some(db) = conn.as_sql() { + windmill_common::partition::set_resolved_partition(db, job.id, &value).await?; + } else { + tracing::warn!( + job_id = %job.id, + "agent worker: resolved partition not persisted; downstream cascade will not propagate it" + ); + } + + // Inject into the in-memory args so the running body sees it. + let mut updated = job.clone(); + let mut map = updated.args.take().map(|j| j.0).unwrap_or_default(); + map.insert( + PARTITION_ARG.to_string(), + windmill_common::worker::to_raw_value(&value), + ); + updated.args = Some(Json(map)); + Ok((Some(updated), in_pipeline)) +} + #[tracing::instrument(level = "trace", skip_all)] async fn handle_code_execution_job( job: &MiniPulledJob, @@ -4456,6 +4567,19 @@ async fn handle_code_execution_job( ), }; + // Pipeline partition resolution: the content is now loaded, so resolve + // `// partitioned` (if any) and shadow `job` with a clone whose args + // carry the resolved `partition` for the rest of execution. + let _job_with_partition; + let (resolved_job, in_pipeline) = resolve_partition_for_job(job, code, conn).await?; + let job = match resolved_job { + Some(j) => { + _job_with_partition = j; + &_job_with_partition + } + None => job, + }; + // For preview jobs, extract modules from args._MODULES if not already set let modules = modules_from_data.clone().or_else(|| { job.args.as_ref().and_then(|args| { @@ -4501,6 +4625,7 @@ async fn handle_code_execution_job( lock, &modules, false, + in_pipeline, ) .await } @@ -4567,6 +4692,9 @@ pub async fn run_language_executor( lock: &Option, modules: &Option>, run_inline: bool, + // Whether the script is a `// pipeline` member (parsed once upstream) — sets + // WM_PIPELINE so the wmll.ducklake SDK helpers record materialization state. + in_pipeline: bool, ) -> error::Result> { // Defense-in-depth (GHSA-wxjq-w5pj-jqhx): the entrypoint override is // interpolated verbatim into a code position of the generated language @@ -4929,6 +5057,11 @@ mount {{ #[allow(unused_mut)] let mut envs = build_envs(envs.as_ref())?; + // Signal pipeline context to the script so the wmll.ducklake SDK helpers + // record materialization state (the grid/backfill) and skip it otherwise. + if in_pipeline { + envs.insert("WM_PIPELINE".to_string(), "true".to_string()); + } let Some(language) = language else { return Err(Error::ExecutionErr( @@ -5714,6 +5847,7 @@ pub fn init_worker_internal_server_inline_utils( &None, &None, true, + false, ) .await }) @@ -5795,6 +5929,7 @@ pub fn init_worker_internal_server_inline_utils( &content_info.lockfile, &content_info.modules, true, + false, ) .await }) diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 86263350c3..7f90f9240e 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1279,7 +1279,11 @@ pub async fn update_flow_status_after_job_completion_internal( }), ) } else { - let inc = if continue_on_error { + // An unrecoverable failure (worker crash/OOM) must reach the error handler + // even on a continue_on_error step, so don't advance the step counter past + // the failed module — otherwise the flow would silently continue to the next + // step and hide the worker death. + let inc = if !unrecoverable && continue_on_error { let retry = current_module .as_ref() .and_then(|x| x.retry.clone()) @@ -1708,7 +1712,16 @@ pub async fn update_flow_status_after_job_completion_internal( _ if stop_early => stop_early_err_msg.is_some() && flow_value.failure_module.is_some(), // if stop_early_err_msg some, we want to trigger the error handler before stopping the flow, if any _ if flow_job.is_canceled() => false, true => !is_last_step, - false if unrecoverable => false, + // An unrecoverable failure (a step killed by a worker crash/OOM and surfaced by + // the zombie handler, or an error raised while updating the flow status itself) + // must not be retried or silently skipped, but it should still trigger the flow's + // error handler: an OOM/worker death is precisely when the error handler is expected + // to run. Continue the flow only to reach the failure module, never to retry. + false if unrecoverable => { + !is_failure_step + && !has_triggered_error_handler + && flow_value.failure_module.is_some() + } false if skip_seq_branch_failure || skip_loop_failures || continue_on_error => { !is_last_step } @@ -2059,6 +2072,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_name, flow_runners, &killpill_rx, + unrecoverable, )) .warn_after_seconds(10) .await @@ -2749,6 +2763,10 @@ pub async fn handle_flow( worker_name: &str, flow_runners: Option>, killpill_rx: &tokio::sync::broadcast::Receiver<()>, + // The previous step failed unrecoverably (e.g. a worker crash/OOM surfaced by the + // zombie handler). The next pushed step can only be the error handler (failure + // module), and it must not be pinned to the dead worker via same_worker. + unrecoverable: bool, ) -> anyhow::Result<()> { let flow = flow_data.value(); @@ -2922,6 +2940,7 @@ pub async fn handle_flow( flow_runners.clone(), job_completed_tx.clone(), &killpill_rx, + unrecoverable, )) .warn_after_seconds(10) .await?; @@ -3104,6 +3123,10 @@ async fn push_next_flow_job( flow_runners: Option>, job_completed_tx: JobCompletedSender, killpill_rx: &tokio::sync::broadcast::Receiver<()>, + // The prior step failed unrecoverably (worker crash/OOM). The only step pushed + // from here is the error handler, which must run on a live worker rather than + // being pinned to the dead one via same_worker / dedicated runners. + unrecoverable: bool, ) -> error::Result { let job_root = flow_job .flow_innermost_root_job @@ -3657,7 +3680,10 @@ async fn push_next_flow_job( } }; - let retry = if matches!(&status_module, FlowStatusModule::Failure { .. },) { + // An unrecoverable failure (worker crash/OOM) must not be retried — the original worker + // and its state are gone — so skip retry evaluation and fall straight through to the + // failure module below. + let retry = if !unrecoverable && matches!(&status_module, FlowStatusModule::Failure { .. },) { let retry = &module.retry.clone().unwrap_or_default(); evaluate_retry( retry, @@ -3672,8 +3698,13 @@ async fn push_next_flow_job( None }; let get_args_from_id = match &status_module { + // `|| unrecoverable`: a worker crash/OOM routes to the failure module even on a + // continue_on_error step (whose failures are normally tolerated), matching the + // `unrecoverable` decision in update_flow_status_after_job_completion_internal. FlowStatusModule::Failure { job, .. } - if retry.as_ref().is_some() || !module.continue_on_error.is_some_and(|x| x) => + if retry.as_ref().is_some() + || !module.continue_on_error.is_some_and(|x| x) + || unrecoverable => { if let Some((fail_count, retry_in)) = retry { tracing::debug!( @@ -4022,7 +4053,8 @@ async fn push_next_flow_job( .as_ref() .is_some_and(|fr| fr.job_id == flow_job.id); - let continue_with_runners = (start_runners || (flow_runners.is_some() && !do_not_pass_runners)) + let continue_with_runners = !unrecoverable + && (start_runners || (flow_runners.is_some() && !do_not_pass_runners)) && module.suspend.is_none() && module.sleep.is_none(); @@ -4031,8 +4063,13 @@ async fn push_next_flow_job( let job_same_worker = flow_job.same_worker && matches!(flow_job.kind, JobKind::Flow) && flow_job.runnable_id.is_some(); - let continue_on_same_worker = - (flow.same_worker || job_same_worker) && module.suspend.is_none() && module.sleep.is_none(); + // After an unrecoverable failure the original worker is gone, so the error handler + // step is pushed as a regular queued job (any live worker can pick it up) instead of + // being signaled to the dead worker via same_worker — which would strand it forever. + let continue_on_same_worker = !unrecoverable + && (flow.same_worker || job_same_worker) + && module.suspend.is_none() + && module.sleep.is_none(); /* Finally, push the job into the queue */ let mut uuids = vec![]; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 3610ad0602..243366b147 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.727.0"; +export const VERSION = "v1.734.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 665021ffdc..5c9353dd21 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -28,6 +28,9 @@ rmSync(outDir, { recursive: true, force: true }); // Build with bun — bundle everything except esbuild (platform-specific binary), // svelte (optional, only needed for `wmill app bundle/dev`), and parser packages // (loaded at runtime via init() with readFileSync for the .wasm binary). +// esbuild-wasm is not a dependency at all: the host/binary-mismatch fallback in +// esbuild_loader.ts downloads and caches the whole esbuild-wasm package at +// runtime, so it stays out of the bundle and the published dependencies. console.log("Bundling with bun build..."); const buildResult = Bun.spawnSync([ "bun", "build", "src/main.ts", diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index e2f8553e22..07ee266145 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -6,6 +6,7 @@ import * as log from "../../core/log.ts"; import { colors } from "@cliffy/ansi/colors"; import * as windmillUtils from "@windmill-labs/shared-utils"; import { readTextFile, readTextFileSync } from "../../utils/utils.ts"; +import { getEsbuild, stopEsbuild } from "../../utils/esbuild_loader.ts"; export interface BundleOptions { entryPoint?: string; outDir?: string; @@ -170,8 +171,9 @@ export async function ensureNodeModules(appDir?: string): Promise { export async function createBundle( options: BundleOptions = {} ): Promise { - // Dynamically import esbuild - const esbuild = await import("esbuild"); + // Native esbuild with a transparent esbuild-wasm fallback on host/binary + // version mismatch (see esbuild_loader.ts). + const esbuild = await getEsbuild(); // Detect frameworks to determine default entry point. // Use the entryPoint's directory if provided, otherwise fall back to cwd. @@ -286,6 +288,10 @@ export async function createBundle( outfile, sourcemap, minify, + // Keep outputs in memory: esbuild-wasm cannot write to the filesystem + // ("write" option unavailable), and the dist files were discarded after the + // read anyway. Native esbuild supports write:false + outputFiles too. + write: false as const, define: { "process.env.NODE_ENV": production ? '"production"' : '"development"', }, @@ -307,29 +313,24 @@ export async function createBundle( log.info(colors.green("✅ Bundle created successfully")); - // Read the generated files - const jsPath = path.join(process.cwd(), outfile); - const cssPath = path.join(process.cwd(), outDir, "bundle.css"); + const outputFiles = result.outputFiles ?? []; + const jsFile = outputFiles.find((f) => f.path.endsWith(".js")); + const cssFile = outputFiles.find((f) => f.path.endsWith(".css")); - if (!fs.existsSync(jsPath)) { - throw new Error(`Expected JS bundle at ${jsPath} but file not found`); + if (!jsFile) { + throw new Error("Expected a JS bundle in esbuild output but none found"); } - const jsContent = readTextFileSync(jsPath); - const cssContent = fs.existsSync(cssPath) - ? readTextFileSync(cssPath) - : ""; - try { fs.rmSync(distDir, { recursive: true }); } catch { //ignore } - return { js: jsContent, css: cssContent }; - + return { js: jsFile.text, css: cssFile?.text ?? "" }; + } finally { - // Stop esbuild - await esbuild.stop(); + // Stop the native esbuild service so the process can exit (no-op for wasm). + await stopEsbuild(); } } diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index c73a8685c8..1a226ec5ef 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -437,7 +437,11 @@ async function dev(opts: DevOptions, appFolder?: string) { const rawApp = (await yamlParseFile(rawAppPath)) as any; const appPath = rawApp?.custom_path ?? "u/unknown/newapp"; - // Dynamically import esbuild only when the dev command is called + // Dynamically import esbuild only when the dev command is called. + // Native-only here (no esbuild-wasm fallback via getEsbuild): dev is a local + // interactive command that relies on context()/watch, whose semantics under + // wasm are untested. The host/binary-mismatch fallback covers the bundling + // paths that run on workers/CI via `wmill sync push`. const esbuild = await import("esbuild"); const host = opts.host ?? DEFAULT_HOST; diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index f790b2b58b..32291616f2 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -118,6 +118,61 @@ const migrateCommand = new Command() ) .action(migrateDown as any); +async function create( + opts: GlobalOptions & { resource?: string; force?: boolean }, + name?: string, +) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dtName = name ?? DEFAULT_DATATABLE_NAME; + + const existing = await wmill.listDataTables({ + workspace: workspace.workspaceId, + }); + if (existing.some((d) => d.name === dtName)) { + throw new Error(`Datatable '${dtName}' already exists in this workspace`); + } + // edit_datatable_config replaces the whole settings object, and fork + // metadata on existing datatables can't be read back through the API — + // so only touch a non-empty config when explicitly asked to. + if (existing.length > 0 && !opts.force) { + throw new Error( + `Workspace already has datatable(s): ${existing + .map((d) => d.name) + .join(", ")}. Re-run with --force to add '${dtName}' ` + + "(note: fork metadata on existing datatables is not preserved)", + ); + } + + const datatables: Record< + string, + { database: { resource_type: "postgresql" | "instance"; resource_path?: string } } + > = {}; + for (const d of existing) { + datatables[d.name] = { + database: { + resource_type: d.resource_type as "postgresql" | "instance", + resource_path: d.resource_path ?? undefined, + }, + }; + } + datatables[dtName] = opts.resource + ? { database: { resource_type: "postgresql", resource_path: opts.resource } } + : { database: { resource_type: "instance", resource_path: "datatable_db" } }; + + await wmill.editDataTableConfig({ + workspace: workspace.workspaceId, + requestBody: { settings: { datatables } }, + }); + log.info( + `Datatable '${dtName}' created (${ + opts.resource + ? `postgresql resource ${opts.resource}` + : "instance-backed" + }). Scripts can now use datatable://${dtName}.`, + ); +} + async function serve( opts: GlobalOptions & { port?: number; host?: string; password?: string }, ) { @@ -147,6 +202,20 @@ const command = new Command() ) .action(run as any) .command("migrate", migrateCommand) + .command( + "create", + "register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://", + ) + .arguments("[name:string]") + .option( + "--resource ", + "Back the datatable with an existing postgresql resource path instead of the instance database", + ) + .option( + "--force", + "Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved)", + ) + .action(create as any) .command( "serve", "Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string", diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index cf909c42a5..7eab79b52d 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -803,7 +803,7 @@ async function rehashCommand( } const command = new Command() - .description("Generate metadata (locks, schemas) for all scripts, flows, and apps") + .description("Regenerate stale local locks and script schemas and refresh wmill-lock.yaml content hashes (scripts, flows, apps). Writes local files only, not a deploy. Run it after edits that add or remove imports or change a script's arguments, so the lock, the auto-generated UI schema, and wmill-lock.yaml stay in sync.") .arguments("[folder:string]") .option("--yes", "Skip confirmation prompt") .option("--dry-run", "Show what would be updated without making changes") @@ -827,9 +827,7 @@ const command = new Command() "rehash", new Command() .description( - "Trust on-disk content; rewrite wmill-lock.yaml hashes without backend " + - "trips or yaml/lock rewrites. Useful for bootstrapping missing lockfile " + - "entries or recovering from older-CLI hash drift." + "Refresh wmill-lock.yaml content hashes from the on-disk .lock and .script.yaml without re-resolving dependencies or hitting the backend. Use when those files are already correct and only the hashes need updating: bootstrapping missing entries or recovering from hash drift." ) .arguments("[folder:string]") .option("--skip-scripts", "Skip processing scripts") diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts new file mode 100644 index 0000000000..b4e08f6416 --- /dev/null +++ b/cli/src/commands/pipeline/pipeline.ts @@ -0,0 +1,287 @@ +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; + +import { OpenAPI } from "../../../gen/index.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; + +// Mirrors the asset-graph endpoint payload (backend/windmill-api-assets). +// TODO: the checked-in generated client (cli/gen, last regenerated 2025-04) +// predates these routes, so we raw-fetch and hand-roll the types. Once +// `cli/gen` is regenerated (run `cli/gen_wm_client.sh`, which is currently +// >700 openapi.yaml commits stale and would churn the whole client), replace +// `apiGet` + these types with the generated `wmill.getAssetsGraph(...)` +// (operationId getAssetsGraph) and `wmill.listPipelineFolders(...)` +// (operationId listPipelineFolders). +type GraphRunnable = { + path: string; + usage_kind: "script" | "flow" | "job"; + in_pipeline?: boolean; +}; +type GraphEdge = { + runnable_kind: string; + runnable_path: string; + asset_kind: string; + asset_path: string; + access_type?: "r" | "w" | "rw"; +}; +type GraphTrigger = + | { + trigger_kind: "asset"; + asset_kind: string; + asset_path: string; + runnable_kind: string; + runnable_path: string; + } + | { + trigger_kind: string; + path?: string; + runnable_kind: string; + runnable_path: string; + missing?: boolean; + }; +type AssetGraph = { + runnables: GraphRunnable[]; + assets: { kind: string; path: string }[]; + edges: GraphEdge[]; + triggers: GraphTrigger[]; +}; + +async function apiGet(path: string): Promise { + const response = await fetch(`${OpenAPI.BASE}${path}`, { + headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` }, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`GET ${path} -> ${response.status}: ${body}`); + } + return (await response.json()) as T; +} + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const items = await apiGet<{ folder: string; script_count: number }[]>( + `/w/${workspace.workspaceId}/assets/pipelines`, + ); + if (opts.json) { + console.log(JSON.stringify(items)); + } else if (items.length === 0) { + log.info( + "No pipelines in this workspace. Mark scripts with a `// pipeline` comment (plus `// on ` triggers) and push them into a folder.", + ); + } else { + new Table() + .header(["Folder", "Scripts"]) + .padding(2) + .border(true) + .body(items.map((p) => [`f/${p.folder}`, String(p.script_count)])) + .render(); + } +} + +const ASSET_KINDS = "s3object,ducklake,datatable,volume"; + +function assetUri(kind: string, path: string): string { + const prefix = kind === "s3object" ? "s3" : kind; + return `${prefix}://${path}`; +} + +function shortName(scriptPath: string): string { + return scriptPath.split("/").pop() ?? scriptPath; +} + +// Append to a multimap value, creating the bucket on first use. Avoids the +// O(n^2) spread-rebuild pattern (`map.set(k, [...(map.get(k) ?? []), v])`). +function pushTo(map: Map, key: K, val: V): void { + (map.get(key) ?? map.set(key, []).get(key)!).push(val); +} + +async function show( + opts: GlobalOptions & { json?: boolean }, + folder: string, +) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const f = folder.replace(/^f\//, "").replace(/\/$/, ""); + const graph = await apiGet( + `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + ); + if (opts.json) { + console.log(JSON.stringify(graph)); + return; + } + if (graph.runnables.length === 0) { + log.info( + `No pipeline scripts in f/${f}. Mark scripts with a \`// pipeline\` comment and push them.`, + ); + return; + } + + // Index the graph: writes per script, subscribers per asset, native + // trigger markers per script, asset subscriptions per script. + const writesByScript = new Map(); + for (const e of graph.edges) { + if (e.access_type === "w" || e.access_type === "rw") { + const uri = assetUri(e.asset_kind, e.asset_path); + pushTo(writesByScript, e.runnable_path, uri); + } + } + const subsByAsset = new Map(); + const subsByScript = new Map(); + const nativeByScript = new Map< + string, + { kind: string; path?: string; missing?: boolean }[] + >(); + for (const t of graph.triggers) { + if (t.trigger_kind === "asset") { + const at = t as Extract; + const uri = assetUri(at.asset_kind, at.asset_path); + pushTo(subsByAsset, uri, t.runnable_path); + pushTo(subsByScript, t.runnable_path, uri); + } else { + const nt = t as Exclude; + pushTo(nativeByScript, t.runnable_path, { + kind: nt.trigger_kind, + path: nt.path, + missing: nt.missing, + }); + } + } + + function triggerBadges(script: string): string { + const out: string[] = []; + for (const t of nativeByScript.get(script) ?? []) { + if (t.kind === "data_upload") { + out.push(colors.magenta("[data upload]")); + } else if (t.missing) { + out.push(colors.red(`[${t.kind} ✗ missing]`)); + } else { + out.push(colors.yellow(`[${t.kind}${t.path ? ` ${t.path}` : ""}]`)); + } + } + return out.length > 0 ? " " + out.join(" ") : ""; + } + + const printed = new Set(); + const lines: string[] = []; + + function printScript(script: string, prefix: string, extraOn?: string[]) { + const alsoOn = + extraOn && extraOn.length > 0 + ? colors.dim(` (also on: ${extraOn.join(", ")})`) + : ""; + if (printed.has(script)) { + lines.push( + `${prefix}${colors.bold(shortName(script))}${colors.dim(" ↻ shown above")}`, + ); + return; + } + printed.add(script); + lines.push(`${prefix}${colors.bold(shortName(script))}${triggerBadges(script)}${alsoOn}`); + const childPrefix = prefix.replace(/├─ $/, "│ ").replace(/└─ $/, " "); + const writes = [...(writesByScript.get(script) ?? [])].sort(); + writes.forEach((uri, i) => { + const lastAsset = i === writes.length - 1; + const assetBranch = lastAsset ? "└─▶ " : "├─▶ "; + lines.push(`${childPrefix}${assetBranch}${colors.cyan(uri)}`); + const assetChildPrefix = childPrefix + (lastAsset ? " " : "│ "); + const subs = [...(subsByAsset.get(uri) ?? [])].sort(); + subs.forEach((sub, j) => { + const branch = j === subs.length - 1 ? "└─ " : "├─ "; + const otherOn = (subsByScript.get(sub) ?? []).filter((u) => u !== uri); + printScript(sub, assetChildPrefix + branch, otherOn); + }); + }); + } + + // Roots: pipeline scripts that aren't subscribed to any asset — sources + // (data upload, schedule, webhook) and manual entries. + const roots = graph.runnables + .map((r) => r.path) + .filter((p) => !(subsByScript.get(p)?.length)) + .sort(); + + // UI-first markers (data_upload, webhook) have no trigger row — the + // graph endpoint's trigger enum (schedule/email/kafka/mqtt/nats/postgres/ + // sqs/gcp) can't surface them, so they only exist as `// on ` + // annotations in the script body. Roots are where sources matter, so fetch + // just those bodies and lift the marker kinds the canvas would show. + // + // DRIFT RISK: this regex + MARKER_KINDS is a divergent, partial copy of the + // canonical annotation parser. The proper fix is to have the graph endpoint + // emit these UI-only markers as trigger rows (a backend change), after which + // this whole Promise.all body-fetch can be deleted and read straight from + // the response. Until then, keep this list in sync with the canonical parser. + const MARKER_KINDS = ["data_upload", "webhook", "email"]; + await Promise.all( + roots.map(async (p) => { + const r = graph.runnables.find((x) => x.path === p); + if (r?.usage_kind !== "script") return; + try { + const script = await wmill.getScriptByPath({ + workspace: workspace.workspaceId, + path: p, + }); + const existing = nativeByScript.get(p) ?? []; + for (const line of (script.content ?? "").split("\n")) { + const m = line.match(/^\s*(?:\/\/|--|#)\s*on\s+(\w+)\s*$/); + if (!m) continue; + const kind = m[1]; + if (!MARKER_KINDS.includes(kind)) continue; + if (!existing.some((t) => t.kind === kind)) { + existing.push({ kind }); + } + } + if (existing.length > 0) nativeByScript.set(p, existing); + } catch { + // body fetch is best-effort enrichment only + } + }), + ); + + const scriptCount = graph.runnables.length; + const assetCount = graph.assets.length; + log.info( + colors.bold(`Pipeline f/${f}`) + + colors.dim(` — ${scriptCount} script${scriptCount === 1 ? "" : "s"} · ${assetCount} asset${assetCount === 1 ? "" : "s"}`), + ); + lines.push(""); + for (const root of roots) { + printScript(root, ""); + lines.push(""); + } + // Anything unreachable from the roots (e.g. cycles) still gets listed. + for (const r of graph.runnables) { + if (!printed.has(r.path)) { + printScript(r.path, ""); + lines.push(""); + } + } + console.log(lines.join("\n")); +} + +const command = new Command() + .description( + "inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` annotations)", + ) + .command("list", "list pipeline folders in the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command( + "show", + "render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal", + ) + .arguments("") + .option("--json", "Output the raw asset graph as JSON") + .action(show as any); + +export default command; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 717e8acf46..ed57b1747d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -58,6 +58,7 @@ import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; import fs from "node:fs"; import { createTarBlob, type TarEntry } from "../../utils/tar.ts"; +import { getEsbuild } from "../../utils/esbuild_loader.ts"; import { execSync } from "node:child_process"; import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts"; @@ -328,7 +329,7 @@ export async function handleFile( }).toString(); log.info("Custom bundler executed for " + path); } else { - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); log.info(`Started bundling ${path} ...`); const startTime = performance.now(); @@ -1565,7 +1566,7 @@ async function preview( maxBuffer: 1024 * 1024 * 50, }).toString(); } else { - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); if (!opts.silent) { log.info(`Bundling ${filePath} for preview...`); diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index b2c6688154..11061f9280 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.727.0"; +export const VERSION = "1.734.0"; diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 25890e1923..cd9ba686ef 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -117,6 +117,20 @@ Local previews exist for every entity type and don't deploy: Argument shapes and per-language details live in the \`write-script-\`, \`write-flow\`, and \`raw-app\` skills. +## Keeping metadata in sync + +After editing a script, flow inline script, or app runnable, its generated metadata can go stale. \`wmill-lock.yaml\` stores a content hash per item, so a change that **adds or removes an import** or **changes a script's arguments** invalidates that hash and leaves the \`.lock\` (resolved dependencies) and \`.script.yaml\` (the input schema that drives the auto-generated args UI) out of date. \`wmill generate-metadata\` regenerates them and refreshes the hashes. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files — it is **not** a deploy — but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default **offer it and run it once the user agrees**, rather than running it silently after every edit. YOU run the command (never tell the user to run it); the choice is only whether to confirm first. + +After running it, diff the regenerated lockfiles (e.g. \`git diff\` the \`.lock\` / \`.script.lock\` files): if any dependency versions changed, tell the user what bumped (e.g. \`requests 2.31.0 → 2.32.0\`) so they can catch an unwanted change before deploying. Do this even under \`Metadata: auto\` — it is information, not a confirmation gate. Pin a version in code to keep it fixed. + +With no path argument it regenerates only the items whose metadata is actually stale (content hash drifted), workspace-wide — not everything. The set can be larger than the file you edited for two reasons: imports propagate (editing a script that others import marks every importer stale too, so their locks regenerate against the new code — by design, since a lock must reflect the imported code), and any pre-existing drift is swept in. If it touches items you didn't expect, run \`wmill generate-metadata --dry-run\` first — it lists each stale item with a reason (\`content changed\` or \`depends on \`) and changes nothing, so you can see why each is in scope. To narrow it, pass a folder or file path (\`wmill generate-metadata f/foo\`); add \`--strict-folder-boundaries\` to touch only items literally inside that folder (it warns about stale importers outside the folder that it skipped — they resurface as stale on the next unscoped run). + +**Save the preference so you don't ask every session.** If the user wants metadata regenerated automatically after edits (or always confirmed first), record it in the **project-specific instructions** section of \`AGENTS.md\` (user-owned — never overwritten by \`wmill refresh prompts\`), e.g. a line like \`Metadata: auto (run wmill generate-metadata after edits)\` or \`Metadata: ask first\`. Read that line first on later sessions and follow it. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ## Deploying There are two ways local changes reach the workspace. Pick based on how the repo is wired, not habit. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index f46ccc992e..1132207239 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -7,6 +7,7 @@ export interface SkillMetadata { } export const SKILLS: SkillMetadata[] = [ + { name: "write-script-ansible", description: "MUST use when writing Ansible playbooks.", languageKey: "ansible" }, { name: "write-script-bash", description: "MUST use when writing Bash scripts.", languageKey: "bash" }, { name: "write-script-bigquery", description: "MUST use when writing BigQuery queries.", languageKey: "bigquery" }, { name: "write-script-bun", description: "MUST use when writing Bun/TypeScript scripts.", languageKey: "bun" }, @@ -38,9 +39,9 @@ export const SKILLS: SkillMetadata[] = [ // Skill content for each skill (loaded inline for bundling) export const SKILL_CONTENT: Record = { - "write-script-bash": `--- -name: write-script-bash -description: MUST use when writing Bash scripts. + "write-script-ansible": `--- +name: write-script-ansible +description: MUST use when writing Ansible playbooks. --- ## CLI Commands @@ -51,7 +52,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -66,13 +67,163 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. + +For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. + +Use \`wmill resource-type list --schema\` to discover available resource types. + +# Ansible + +Windmill runs Ansible playbooks with \`ansible-playbook\`. A script is a single YAML +document made of two parts separated by a \`---\` line: a Windmill **header** and one or +more standard Ansible **plays**. + +## Structure + +\`\`\`yaml +--- +# Windmill header: configures inventories, file resources, arguments and dependencies +extra_vars: + world_qualifier: + type: string +dependencies: + galaxy: + collections: + - name: community.general + python: + - jmespath +--- +# Standard Ansible plays +- name: Echo + hosts: 127.0.0.1 + connection: local + tasks: + - name: Print debug message + debug: + msg: "Hello, {{ world_qualifier }} world!" +\`\`\` + +## Header + +The header is **not** standard Ansible — it is parsed by Windmill to build the script's +inputs and runtime environment. Supported keys: + +- \`extra_vars\`: defines the script arguments. Each entry is passed to the playbook via + \`--extra-vars\` and becomes a Jinja variable usable as \`{{ name }}\` in the plays. Give + each argument a \`type\` (\`string\`, \`number\`, \`boolean\`, \`object\`, ...) so Windmill can + generate the input form. +- \`inventory\`: lists inventories. Use \`resource_type: ansible_inventory\` (optionally + pinned with \`resource: u/user/your_resource\`) or \`resource_type: dynamic_inventory\`. +- \`files\`: writes Windmill resources/variables to files before the run, e.g. + \`- resource: u/user/template\` with \`target: ./config.j2\`, or + \`- variable: u/user/ssh_key\` with \`target: ./ssh_key\` and \`mode: '0600'\`. +- \`dependencies\`: \`galaxy\` collections/roles (installed with \`ansible-galaxy\`) and + \`python\` pip packages available to the playbook. +- \`options\`: extra \`ansible-playbook\` flags such as \`- verbosity: vvv\`. +- \`vault_password\`: a Windmill variable path to use as the Ansible Vault password. + +## Arguments + +Reference header \`extra_vars\` directly as Jinja variables in the plays: + +\`\`\`yaml +extra_vars: + name: + type: string + count: + type: number +--- +- hosts: localhost + tasks: + - debug: + msg: "{{ name }} x {{ count }}" +\`\`\` + +## Environment variables + +Windmill contextual variables are available as environment variables and read with the +\`env\` lookup: + +\`\`\`yaml +- debug: + msg: "Running in workspace {{ lookup('env', 'WM_WORKSPACE') }}" +\`\`\` + +## Output + +To return a result, write JSON to a \`result.json\` file in the job directory: + +\`\`\`yaml +- hosts: localhost + tasks: + - name: Write result + copy: + content: "{{ { 'ok': true, 'value': 42 } | to_json }}" + dest: result.json +\`\`\` +`, + "write-script-bash": `--- +name: write-script-bash +description: MUST use when writing Bash scripts. +--- + +## CLI Commands + +Place scripts in a folder. + +After writing, tell the user which command fits what they want to do: + +- \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. +- \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. +- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". + +### Preview vs run — choose by intent, not habit + +If the user says "run the script", "try it", "test it", "does it work" while there are **local edits to the script file**, use \`script preview\`. Do NOT push the script to then \`script run\` it — pushing is a deploy, and deploying just to test overwrites the workspace version with untested changes. + +Only use \`script run\` when: +- The user explicitly says "run the deployed version" / "run what's on the server". +- There is no local script being edited (you're just invoking an existing script). + +Only use \`sync push\` when: +- The user explicitly asks to deploy, publish, push, or ship. +- The preview has already validated the change and the user wants it in the workspace. + +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + +### After writing — offer to test, don't wait passively + +If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. + +If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. + +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -141,7 +292,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -156,13 +307,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -224,7 +385,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -239,13 +400,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -916,7 +1087,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") + * @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main") * @returns SQL template function for building parameterized queries * @example * let sql = wmill.ducklake() @@ -926,8 +1097,34 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * SELECT * FROM friends * WHERE name = \${name} AND age = \${age} * \`.fetch() + * @example + * // Target a specific schema within the ducklake + * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-bunnative": `--- name: write-script-bunnative @@ -942,7 +1139,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -957,13 +1154,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -1634,7 +1841,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") + * @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main") * @returns SQL template function for building parameterized queries * @example * let sql = wmill.ducklake() @@ -1644,8 +1851,34 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * SELECT * FROM friends * WHERE name = \${name} AND age = \${age} * \`.fetch() + * @example + * // Target a specific schema within the ducklake + * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-csharp": `--- name: write-script-csharp @@ -1660,7 +1893,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -1675,13 +1908,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -1742,7 +1985,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -1757,13 +2000,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2434,7 +2687,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction /** * Create a SQL template function for DuckDB/ducklake queries - * @param name - DuckDB database name (default: "main") + * @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main") * @returns SQL template function for building parameterized queries * @example * let sql = wmill.ducklake() @@ -2444,8 +2697,34 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction * SELECT * FROM friends * WHERE name = \${name} AND age = \${age} * \`.fetch() + * @example + * // Target a specific schema within the ducklake + * let sql = wmill.ducklake("my_lake:analytics") */ ducklake(name: string = "main"): SqlTemplateFunction + +/** + * Idempotently materialize \`selectSql\` into a ducklake table for one + * partition (or the whole table when \`partition\` is omitted) — the client-side + * equivalent of the \`// materialize\` engine. + * With \`uniqueKey\` it upserts the slice (delete-by-key + insert); otherwise it + * replaces it (whole table → \`CREATE OR REPLACE\`; partition → delete + insert). + * Safe to re-run for the same partition (backfill / failure-recovery). + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.upsertPartition({ table, selectSql, partition }).execute()\`. + */ +upsertPartition(opts: DucklakeMaterializeOptions): SqlStatement + +/** + * INSERT-only materialization (no dedup/replace) for append-only tables. + * Re-running the same partition duplicates rows — use only for immutable + * event-log sources. + * + * Returns a lazy statement — call \`.execute()\` to run it: + * \`await wmill.appendPartition({ table, selectSql, partition }).execute()\`. + */ +appendPartition(opts: Omit,): SqlStatement `, "write-script-duckdb": `--- name: write-script-duckdb @@ -2460,7 +2739,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2475,13 +2754,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2576,7 +2865,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2591,13 +2880,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2675,7 +2974,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2690,13 +2989,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2761,7 +3070,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2776,13 +3085,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2840,7 +3159,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2855,13 +3174,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2922,7 +3251,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2937,13 +3266,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3005,7 +3344,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3020,13 +3359,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3103,7 +3452,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3118,13 +3467,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3184,7 +3543,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3199,13 +3558,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3280,7 +3649,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3295,13 +3664,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3996,6 +4375,27 @@ def stream_result(stream) -> None # SqlQuery instance for fetching results def query(sql: str, *args) -> SqlQuery +# Idempotently materialize the rows of \`select_sql\` into ducklake +# \`table\` for one \`partition\` (or the whole table when \`partition\` is +# None). Client-side equivalent of the \`// materialize\` engine: with +# \`unique_key\` it upserts within the slice (delete-by-key + insert); +# without it, it replaces (whole table → CREATE OR REPLACE; partition → +# delete the partition + insert). Re-running the same slice is safe — the +# backfill / failure-recovery contract. +# +# The partition value is bound as a DuckDB arg (never string-interpolated) +# so it cannot inject SQL. \`select_sql\` is trusted (your own query). +def upsert_partition(table: str, select_sql: str, partition: str = None, unique_key: str = None, partition_col: str = '_wm_partition', schema: str = None) + +# INSERT-only materialization (no dedup / no replace) for an immutable +# event-log table — for one \`partition\`, or the whole table when +# \`partition\` is None. NOTE: unlike \`upsert_partition\`, re-running the same +# slice duplicates rows — use only for append-only sources. +def append_partition(table: str, select_sql: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None) + +# Read a materialized ducklake table, optionally a single partition. +def read(table: str, partition: str = None, partition_col: str = '_wm_partition', schema: str = None) + # Execute query and fetch results. # # Args: @@ -4146,7 +4546,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4161,13 +4561,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4272,7 +4682,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4287,13 +4697,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4388,7 +4808,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4403,13 +4823,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4504,11 +4934,11 @@ Once the flow has real content, **offer** to open the visual preview as a one-se ## CLI Commands — running, previewing, deploying -After writing, act on the user's intent instead of just listing commands. Run the safe, non-deploying command yourself when it fits (\`wmill flow preview\` — see "After writing — offer to run, don't wait passively" below); only *name* the commands that deploy or rewrite files (\`wmill sync push\`, \`wmill generate-metadata\`) so the user can approve them. The options: +After writing, act on the user's intent instead of just listing commands. Run \`wmill flow preview\` yourself when it fits (see "After writing — offer to run, don't wait passively" below). \`wmill generate-metadata\` regenerates local lock/hash files (not a deploy) but re-resolves deps — offer it and run on agreement, unless the project's \`AGENTS.md\` opts into running metadata automatically. Only *name* \`wmill sync push\` (the deploy) so the user can approve it. The options: - \`wmill flow preview \` — **default when iterating on a local flow.** Runs the local \`flow.yaml\` against local inline scripts without deploying. Add \`--remote\` to use deployed workspace scripts for PathScript steps instead of local files. Add \`--step \` to run only one module in isolation (see "Single-step vs whole-flow preview" below). - \`wmill flow run \` — runs the flow **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — regenerate stale \`.lock\` and \`.script.yaml\` files. By default it scans **scripts, flows, and apps** across the workspace; pass \`--skip-flows --skip-apps\` (or run from a subdirectory) to limit the scope when you only care about the flow you edited. +- \`wmill generate-metadata\` — regenerate stale local \`.lock\` files for the flow and its inline scripts and refresh their content hashes in \`wmill-lock.yaml\`. Writes local files only (not a deploy). Run it after editing inline scripts whose imports or arguments changed, so \`wmill-lock.yaml\` doesn't drift and add noise to git-sync/CI. By default it scans **scripts, flows, and apps** across the workspace but only regenerates stale ones; pass the flow's folder as an argument (or run from that subdirectory) to limit the scope to the flow you edited. Note a flow (or script) that imports a changed shared script is pulled in too — run \`wmill generate-metadata --dry-run\` to see exactly what is stale and why (\`content changed\` vs \`depends on \`) before applying. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4537,7 +4967,7 @@ If the user hasn't already told you to run/test the flow, offer it as a one-sent If the user already asked to test/run/try the flow in their original request, skip the offer and just execute \`wmill flow preview -d ''\` directly — pick plausible args from the flow's input schema. -\`wmill flow preview\` is safe to run yourself (it does not deploy). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill flow preview\` is safe to run yourself (it does not deploy). \`wmill generate-metadata\` does not deploy either (it only writes local lock/hash files) but re-resolves deps — offer it and run on agreement, unless the project's \`AGENTS.md\` opts into automatic metadata. After running it, check the regenerated \`.lock\` diff and tell the user which inline-script dependency versions changed, so they can catch an unwanted bump before deploying. Only \`wmill sync push\` deploys; run it only when the user explicitly asks. ### Visual preview @@ -4970,10 +5400,11 @@ The runnable ID is the filename without extension. For example, \`get_user.ts\` | C# | \`.cs\` | \`myFunc.cs\` | | Java | \`.java\` | \`myFunc.java\` | -After creating a runnable, offer to generate its lock files as a one-sentence next step (e.g. "Want me to generate the lock files?") and run it yourself once they agree — don't just name the command and wait. If the user already asked you to finish/lock the app, run it directly. It writes local lock files (not a deploy), so offer rather than running silently: +After creating or editing a backend runnable — especially when its imports or arguments changed — its local lock and \`wmill-lock.yaml\` go stale. Offer to run \`wmill generate-metadata\` and run it once the user agrees (or automatically if the project's \`AGENTS.md\` opts into that) — YOU run it, don't just name it and wait. It writes local files only (not a deploy), and keeping the lock current avoids noise in git-sync/CI: \`\`\`bash wmill generate-metadata \`\`\` +After it runs, check the regenerated \`.lock\` diff and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying. ### Optional YAML configuration @@ -5063,7 +5494,7 @@ data: Two commands you run yourself, not the user: - \`wmill app new\` — run it with flags, per the "Creating a Raw App" section above. -- \`wmill generate-metadata\` — generates local lock files; offer it and run it on consent, per "After creating a runnable" above (it writes local lock files, not a deploy). +- \`wmill generate-metadata\` — (re)generates local lock files and refreshes \`wmill-lock.yaml\` content hashes; writes local files only (not a deploy). After adding or editing a runnable, offer it and run it on agreement — or automatically if the project's \`AGENTS.md\` opts into that (see "After creating a runnable" above). For the rest, tell the user which command fits their intent and let them run it — these deploy to the workspace, overwrite local files, or launch a long-running server, so the user should consent each time: @@ -5603,7 +6034,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- \`wmill generate-metadata\` — generate \`.script.yaml\` and \`.lock\` files for the script you modified. +- \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. - \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -5618,13 +6049,23 @@ Only use \`sync push\` when: - The user explicitly asks to deploy, publish, push, or ship. - The preview has already validated the change and the user wants it in the workspace. +### Keep metadata in sync after editing + +\`wmill-lock.yaml\` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing \`main\`'s arguments** — invalidates that hash and leaves the \`.lock\`, the \`.script.yaml\` input schema, and the hash row out of date. Run \`wmill generate-metadata\` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by \`.script.yaml\`), and \`wmill-lock.yaml\` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's \`AGENTS.md\` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated \`.lock\` / \`.script.lock\` files and tell the user which dependency versions changed (e.g. \`requests 2.31.0 → 2.32.0\`), so they can catch an unwanted bump before deploying — even under \`Metadata: auto\`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, \`generate-metadata\` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run \`wmill generate-metadata --dry-run\` — it lists each stale item with a reason (\`content changed\` or \`depends on \`) without changing anything — then narrow with a path argument (\`wmill generate-metadata f/foo\`) or \`--strict-folder-boundaries\`. + +If the on-disk \`.lock\` and \`.script.yaml\` are already correct and only \`wmill-lock.yaml\` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use \`wmill generate-metadata rehash\` — it re-records hashes from disk with no backend round-trip and no dependency changes. + ### After writing — offer to test, don't wait passively If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run \`wmill script preview\` with sample args?"). Do not present a multi-option menu. If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill sync push\` and \`wmill generate-metadata\` modify workspace state or local files — only run these when the user explicitly asks; otherwise tell them which to run. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -6121,6 +6562,9 @@ manage datatable migrations - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. - \`datatable migrate\` +- \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// + - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database + - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) - \`datatable serve\` - Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string - \`--port \` - Port to listen on (default: first free port in 5433-5500) - \`--host \` - Bind address (default: 127.0.0.1) @@ -6233,7 +6677,7 @@ folder related commands ### generate-metadata -Generate metadata (locks, schemas) for all scripts, flows, and apps +Regenerate stale local locks and script schemas and refresh wmill-lock.yaml content hashes (scripts, flows, apps). Writes local files only, not a deploy. Run it after edits that add or remove imports or change a script's arguments, so the lock, the auto-generated UI schema, and wmill-lock.yaml stay in sync. **Arguments:** \`[folder:string]\` @@ -6252,7 +6696,7 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps **Subcommands:** -- \`generate-metadata rehash [folder:string]\` +- \`generate-metadata rehash [folder:string]\` - Refresh wmill-lock.yaml content hashes from the on-disk .lock and .script.yaml without re-resolving dependencies or hitting the backend. Use when those files are already correct and only the hashes need updating: bootstrapping missing entries or recovering from hash drift. - \`--skip-scripts\` - Skip processing scripts - \`--skip-flows\` - Skip processing flows - \`--skip-apps\` - Skip processing apps @@ -6360,7 +6804,7 @@ sync local with a remote instance or the opposite (push or pull) - \`-o, --output-file \` - Write YAML to a file instead of stdout - \`--show-secrets\` - Include sensitive fields (license key, JWT secret) without prompting - \`--instance \` - Name of the instance, override the active instance -- \`instance connect-slack\` +- \`instance connect-slack\` - Non-interactively connect Slack at the instance level using a pre-minted bot token (xoxb-...). Produces the same artifacts as the UI OAuth flow: global_settings 'slack' row + encrypted f/slack_bot/global_bot_token variable and resource in the admins workspace. - \`--bot-token \` - Slack bot token (xoxb-...) - \`--team-id \` - Slack team id - \`--team-name \` - Slack team name @@ -6414,6 +6858,8 @@ Validate Windmill flow, schedule, and trigger YAML files in a directory ### object-storage +Object storage (S3) related commands. Operates on the workspace's default object storage; use --storage to target a configured secondary storage. + **Alias:** \`s3\` **Subcommands:** @@ -6448,8 +6894,21 @@ Validate Windmill flow, schedule, and trigger YAML files in a directory - \`--csv-separator \` - CSV column separator (default ,) - \`--csv-header\` - Treat the first CSV row as a header +### pipeline + +inspect asset-driven pipelines (scripts marked \`// pipeline\`, wired by \`// on \` annotations) + +**Subcommands:** + +- \`pipeline list\` - list pipeline folders in the workspace + - \`--json\` - Output as JSON (for piping to jq) +- \`pipeline show \` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal + - \`--json\` - Output the raw asset graph as JSON + ### protection-rules +Sync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'. + **Subcommands:** - \`protection-rules pull [workspace:string]\` - Pull protection rules from Windmill into protection-rules.yaml for a workspace @@ -6790,7 +7249,7 @@ workspace related commands - \`--bot-token \` - Slack bot token (xoxb-...) - \`--team-id \` - Slack team id - \`--team-name \` - Slack team name -- \`workspace disconnect-slack\` +- \`workspace disconnect-slack\` - Clear slack_team_id / slack_name on the active workspace (marks the workspace as disconnected). Does NOT remove the bot token variable/resource/folder/group — delete those from the local sync folder and run 'wmill sync push' to tear them down. Does NOT remove the workspace-level OAuth override — set slack_oauth_client_id/_secret to '' in settings.yaml and push. diff --git a/cli/src/main.ts b/cli/src/main.ts index efa7df963b..ff8c576ee9 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -51,6 +51,7 @@ import generateMetadata from "./commands/generate-metadata/generate-metadata.ts" import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; import datatable from "./commands/datatable/datatable.ts"; +import pipeline from "./commands/pipeline/pipeline.ts"; import ducklake from "./commands/ducklake/ducklake.ts"; import objectStorage from "./commands/object-storage/object-storage.ts"; import { fetchVersion } from "./core/context.ts"; @@ -77,6 +78,7 @@ export { docs, config, datatable, + pipeline, ducklake, objectStorage, hubPull, @@ -215,6 +217,7 @@ const command = new Command() .command("docs", docs) .command("config", config) .command("datatable", datatable) + .command("pipeline", pipeline) .command("ducklake", ducklake) .command("object-storage", objectStorage) .command("version --version", "Show version information") diff --git a/cli/src/utils/esbuild_loader.ts b/cli/src/utils/esbuild_loader.ts new file mode 100644 index 0000000000..4801c51a46 --- /dev/null +++ b/cli/src/utils/esbuild_loader.ts @@ -0,0 +1,228 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import process from "node:process"; +import { createGunzip } from "node:zlib"; +import { Readable } from "node:stream"; +import { pathToFileURL } from "node:url"; +import * as tar from "tar-stream"; +import * as log from "../core/log.ts"; + +// esbuild splits into a JS host package and a per-platform native binary +// (@esbuild/). They must be the same version. A broken or incremental +// install can leave the on-disk binary at a different version than the pinned +// host, which crashes service start with +// Cannot start service: Host version "X" does not match binary version "Y" +// The running code can't fix what npm/bun put on disk, so when that happens we +// fall back to esbuild-wasm, whose binary is a single version-pinned .wasm. To +// keep that 14MB out of every CLI install, the esbuild-wasm package is not a +// dependency: it is downloaded once and cached on disk, on the fallback path +// only. We download the whole package (not just the .wasm) because esbuild-wasm +// reads the app's files from disk by spawning `node bin/esbuild`, which needs +// bin/esbuild + esbuild.wasm + wasm_exec*.js co-located on disk. + +type Esbuild = typeof import("esbuild"); + +// Version to fall back to if the native host's version can't be read. Keep in +// sync with the "esbuild" pin in cli/package.json. +const FALLBACK_VERSION = "0.28.0"; + +let cached: Esbuild | undefined; +let inFlight: Promise | undefined; +// Distinguishes concurrent extraction temp dirs within a process. +let extractCounter = 0; + +/** + * Returns a working esbuild module, preferring the native binary and falling + * back to esbuild-wasm only when the native host/binary versions don't match. + * Memoized for the process: concurrent first callers (e.g. a parallel + * `wmill sync push`) share one probe/download instead of each running their own. + */ +export function getEsbuild(): Promise { + if (cached) return Promise.resolve(cached); + if (inFlight) return inFlight; + inFlight = acquireEsbuild() + .then((esbuild) => { + cached = esbuild; + return esbuild; + }) + .finally(() => { + inFlight = undefined; + }); + return inFlight; +} + +async function acquireEsbuild(): Promise { + // Escape hatch: skip native entirely (e.g. a host known to have a broken + // install, or to exercise the fallback path). + if (process.env.WINDMILL_FORCE_ESBUILD_WASM) { + return loadWasmEsbuild(await nativeHostVersion()); + } + + try { + const esbuild = await import("esbuild"); + // The native service only starts on the first call; force it with the most + // trivial op so any breakage (host/binary version mismatch, a dead service) + // surfaces now rather than mid-build. The mismatch detail is printed to the + // child's stderr while the thrown error is generic ("service was stopped"), + // so we fall back on ANY smoke-test failure rather than matching a string. + await esbuild.transform(""); + return esbuild; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + log.warn( + `native esbuild is not usable; falling back to esbuild-wasm (${msg.trim()})` + ); + } + + return loadWasmEsbuild(await nativeHostVersion()); +} + +/** + * Stops the esbuild service (native or wasm — both spawn a child process) so the + * process can exit. Safe to call repeatedly; the service restarts lazily on the + * next build. + */ +export async function stopEsbuild(): Promise { + await cached?.stop(); +} + +async function nativeHostVersion(): Promise { + try { + return (await import("esbuild")).version ?? FALLBACK_VERSION; + } catch { + return FALLBACK_VERSION; + } +} + +async function loadWasmEsbuild(version: string): Promise { + const pkgDir = await ensureWasmPackage(version); + const mainJs = path.join(pkgDir, "lib", "main.js"); + // The Node build (lib/main.js) reads app files from disk by spawning + // `node bin/esbuild`, so it works with on-disk entry points and node_modules, + // unlike the browser build. + return (await import(pathToFileURL(mainJs).href)) as unknown as Esbuild; +} + +/** + * Returns a directory containing an extracted esbuild-wasm package (with + * lib/main.js). Uses an explicit override, then an on-disk cache, then downloads + * and extracts the npm tarball. + */ +async function ensureWasmPackage(version: string): Promise { + // Explicit local override wins (air-gapped / self-hosted workers): a path to + // an already-extracted esbuild-wasm package directory. + const override = process.env.WINDMILL_ESBUILD_WASM_PATH; + if (override) return override; + + const destDir = path.join(cacheDir(), `esbuild-wasm-${version}`); + if (fs.existsSync(path.join(destDir, "lib", "main.js"))) { + return destDir; + } + + const url = process.env.WINDMILL_ESBUILD_WASM_URL ?? + `https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-${version}.tgz`; + log.info(`Downloading esbuild-wasm@${version} from ${url} ...`); + const res = await fetch(url); + if (!res.ok || !res.body) { + throw new Error( + `Failed to download esbuild-wasm@${version} (${res.status} ${res.statusText}). ` + + `Set WINDMILL_ESBUILD_WASM_PATH to an extracted esbuild-wasm package dir, ` + + `point WINDMILL_ESBUILD_WASM_URL at a reachable tarball, or repair the native esbuild install.` + ); + } + + // Extract to a unique temp dir and rename into place so a crash or a + // concurrent writer can't leave a half-extracted package behind, and so two + // extractions never share an in-progress directory. + const tmpDir = `${destDir}.${process.pid}.${extractCounter++}.tmp`; + fs.rmSync(tmpDir, { recursive: true, force: true }); + await extractTarball(res.body, tmpDir); + if (!fs.existsSync(path.join(tmpDir, "lib", "main.js"))) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + throw new Error(`esbuild-wasm@${version} tarball did not contain lib/main.js`); + } + try { + fs.renameSync(tmpDir, destDir); + } catch { + // Another process won the race, or rename across devices failed; clean up + // and let the existsSync check below decide whether the cache is usable. + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + if (!fs.existsSync(path.join(destDir, "lib", "main.js"))) { + throw new Error(`Failed to cache esbuild-wasm@${version} at ${destDir}`); + } + return destDir; +} + +/** + * Resolves a tar entry to an absolute path inside destDir, stripping the leading + * "package/" component that npm tarballs use. Returns null if the entry would + * escape destDir (tar-slip), since WINDMILL_ESBUILD_WASM_URL allows untrusted + * tarball sources. + */ +export function resolveTarEntryPath( + destDir: string, + entryName: string +): string | null { + const rel = entryName.replace(/^[^/]+\//, ""); + const root = path.resolve(destDir); + const outPath = path.resolve(root, rel); + if (outPath !== root && !outPath.startsWith(root + path.sep)) { + return null; + } + return outPath; +} + +// Extracts an npm tarball (gzipped tar) into destDir, stripping the leading +// "package/" path component that npm tarballs use. +async function extractTarball( + body: ReadableStream, + destDir: string +): Promise { + const extract = tar.extract(); + extract.on("entry", (header, stream, next) => { + if (header.type !== "file") { + stream.resume(); + stream.on("end", next); + return; + } + const outPath = resolveTarEntryPath(destDir, header.name); + if (!outPath) { + // Reject tar-slip entries that would write outside the cache dir. + stream.resume(); + stream.on("end", () => + next(new Error(`unsafe path in esbuild-wasm tarball: ${header.name}`)) + ); + return; + } + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + const ws = fs.createWriteStream(outPath, { mode: header.mode ?? 0o644 }); + stream.pipe(ws); + ws.on("finish", next); + ws.on("error", next); + stream.on("error", next); + }); + + await new Promise((resolve, reject) => { + extract.on("finish", resolve); + extract.on("error", reject); + Readable.fromWeb(body as unknown as Parameters[0]) + .pipe(createGunzip()) + .on("error", reject) + .pipe(extract) + .on("error", reject); + }); +} + +function cacheDir(): string { + const explicit = process.env.WINDMILL_CACHE_DIR; + if (explicit) return explicit; + const xdg = process.env.XDG_CACHE_HOME; + if (xdg) return path.join(xdg, "windmill"); + try { + return path.join(os.homedir(), ".cache", "windmill"); + } catch { + return path.join(os.tmpdir(), "windmill"); + } +} diff --git a/cli/src/utils/local_path_scripts.ts b/cli/src/utils/local_path_scripts.ts index c9d321dd55..9e03db8a5f 100644 --- a/cli/src/utils/local_path_scripts.ts +++ b/cli/src/utils/local_path_scripts.ts @@ -1,6 +1,7 @@ import { execFileSync } from "node:child_process"; import { stat } from "node:fs/promises"; import { readTextFile } from "./utils.ts"; +import { getEsbuild } from "./esbuild_loader.ts"; import type { SyncCodebase } from "./codebase.ts"; import { parseMetadataFileIfExists } from "./metadata.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; @@ -43,7 +44,7 @@ async function bundleSingleFileCodebaseScript( ).toString(); } - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); const out = await esbuild.build({ entryPoints: [filePath], // Inline rawscripts are executed through the standard module wrapper, diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index f314fe0e5d..a713e41782 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -110,8 +110,16 @@ export function inferContentTypeFromFilePath( return "rlang"; // for related places search: ADD_NEW_LANG } else { + const ext = contentPath.substring(contentPath.lastIndexOf(".")); + let hint = ""; + if (ext === ".sql") { + hint = + "\nBare .sql is ambiguous — use a dialect extension: .pg.sql (postgresql), .my.sql (mysql), .bq.sql (bigquery), .sf.sql (snowflake), .ms.sql (mssql), .odb.sql (oracledb), .duckdb.sql (duckdb)"; + } throw new Error( - "Invalid language: " + contentPath.substring(contentPath.lastIndexOf(".")) + `Cannot infer script language from extension '${ext}' (file ${contentPath}).` + + hint + + "\nSupported extensions: .ts (bun/deno), .py, .go, .sh, .ps1, .php, .rs, .cs, .nu, .java, .rb, .r, .gql, .playbook.yml, .pg.sql, .my.sql, .bq.sql, .sf.sql, .ms.sql, .odb.sql, .duckdb.sql" ); } } diff --git a/cli/test/esbuild_loader_unit.test.ts b/cli/test/esbuild_loader_unit.test.ts new file mode 100644 index 0000000000..9e75607dbf --- /dev/null +++ b/cli/test/esbuild_loader_unit.test.ts @@ -0,0 +1,31 @@ +/** + * Unit tests for esbuild_loader pure logic (no backend, no network). + */ + +import { expect, test, describe } from "bun:test"; +import { resolveTarEntryPath } from "../src/utils/esbuild_loader.ts"; +import { sep, resolve } from "node:path"; + +describe("resolveTarEntryPath", () => { + const dest = resolve("/tmp/cache/esbuild-wasm-0.28.0"); + + test("strips the leading package/ component and resolves inside dest", () => { + expect(resolveTarEntryPath(dest, "package/lib/main.js")).toBe( + dest + sep + "lib" + sep + "main.js" + ); + expect(resolveTarEntryPath(dest, "package/esbuild.wasm")).toBe( + dest + sep + "esbuild.wasm" + ); + }); + + test("rejects tar-slip entries that escape the dest dir", () => { + expect(resolveTarEntryPath(dest, "package/../../etc/passwd")).toBeNull(); + expect(resolveTarEntryPath(dest, "package/../../../outside")).toBeNull(); + }); + + test("rejects entries that only share a prefix with dest", () => { + // ".../esbuild-wasm-0.28.0-evil" must not be treated as inside dest + expect(resolveTarEntryPath(dest, "evil/../../esbuild-wasm-0.28.0-evil/x")) + .toBeNull(); + }); +}); diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md new file mode 100644 index 0000000000..3b5c95c770 --- /dev/null +++ b/docs/ducklake-materialization.md @@ -0,0 +1,250 @@ +# DuckLake-native materialization + +Design sketch for "managed, versioned, incremental" assets built on the +DuckLake substrate. This is a companion to [`pipelines-vs-dbt.md`](./pipelines-vs-dbt.md) +and extends its **Path C (hybrid, partition-first)** recommendation. The new +contribution here is leveraging DuckLake's snapshot/time-travel layer, which +the earlier doc's incremental deep-dive did not use. The annotation grammar is +reconciled with that doc — `// partitioned` + `// unique_key` + `// append` +stay canonical; nothing here forks a competing vocabulary. + +## The core reframe + +dbt had to *build* a materialization engine (compile SQL → `CREATE TABLE AS` / +incremental `MERGE` / SCD2 snapshots) because the warehouse gives it nothing +but raw SQL over a mutable table. DuckLake hands us, at the storage layer, the +four things that engine exists to provide: + +| Capability | Source | +|---|---| +| ACID multi-statement transactions | DuckLake | +| A snapshot per commit + time-travel (`AT (VERSION => n)` / `AT (TIMESTAMP => ...)`) | DuckLake | +| Physical partitioning + pruning (`ALTER TABLE … SET PARTITIONED BY (…)`) | DuckLake | +| Schema evolution tracked in the catalog DB | DuckLake | + +Windmill already attaches DuckLake fully — see `transform_attach_ducklake` +(`backend/windmill-worker/src/duckdb_executor.rs:661`), which rewrites a user's +`ATTACH 'ducklake://name' AS dl` into the real +`ATTACH 'ducklake:postgres:…' AS dl (DATA_PATH 's3://…', OVERRIDE_DATA_PATH +TRUE, AUTOMATIC_MIGRATION TRUE)` at `duckdb_executor.rs:730`. But today every +write is a destructive overwrite and **all four capabilities above are thrown +away** — snapshots are never surfaced, partitioning is purely an orchestration +concept disconnected from the physical layout. + +So the materialization engine we need is a thin layer — a write-strategy +wrapper + snapshot capture — not a dbt rebuild. This is exactly the +"buy A's 80% without B's dialect-rewriting tax" tradeoff `pipelines-vs-dbt.md` +argued for; DuckLake is what makes the remaining 20% (versioning, reproducible +reads, materialization history) nearly free instead of a second project. + +## Annotation grammar (final) + +One self-documenting line; managed-by-default. Strategy options live *on* the +`materialize` line (they have no meaning without it), while `// partitioned` +stays separate because it is cross-cutting (cascade + scheduling + materialize). + +``` +// materialize ducklake://analytics/orders_daily → managed, replace (default) +// materialize ducklake://analytics/orders_daily key=order_id → managed, merge +// materialize ducklake://analytics/orders_daily append → managed, append +// materialize manual ducklake://analytics/orders_daily → track-only escape hatch +``` + +- **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill + generates the write DDL, captures the DuckLake snapshot, and records state. + DuckDB-only; validated at deploy (a non-SELECT script is rejected with a clear + error pointing to the `wmll.ducklake` helpers). +- **`manual`** — escape hatch: the script writes its own DDL; Windmill only + records state (no snapshot capture, no idempotency guarantee). Rare; explicit. +- **`key=`** → MERGE (dedup within slice); **`append`** → INSERT-only; + neither → DELETE-by-partition + INSERT (replace). `append` wins over `key` if + both are given (deploy warning). +- **`// partitioned `** — unit of work + state + backfill (separate; + cross-cutting). Polyglot / multi-statement writes use the `wmll.ducklake` + helpers instead of `// materialize`. + +There is no `wrap` keyword — `materialize` *is* "manage the write," so it was +redundant; the only reason for it was to carve out the weak track-only mode, +which is now the explicit `manual` opt-out. + +DuckLake snapshots are **orthogonal to all of the above** — they apply to every +strategy automatically because every write is a DuckLake commit. The user never +annotates for versioning; they get it. + +## Executor codegen + +### The seam + +`run_duckdb` already splits the script into statement blocks and rewrites +custom `ATTACH` blocks in a single pass before execution +(`duckdb_executor.rs:114-160`): + +```rust +let query_block_list = parse_sql_blocks(&query, true); +// each block: remove_comments → if ducklake/datatable ATTACH, expand; else passthrough +``` + +All blocks run in order on one DuckDB connection. Materialize has two modes: + +1. **Managed (default).** The user writes *setup + one trailing `SELECT`*. Windmill + replaces that SELECT with generated statements, wrapped in an *explicit + DuckLake transaction it controls* — never textual `BEGIN/COMMIT` injected + around the user's other statements (fragile across their own `ATTACH`s and + multi-statement SQL). For a partitioned `replace` the SELECT block expands to: + + ```sql + -- generated for: // partitioned daily ; target = dl.orders_daily ; partition = '2026-06-19' + CREATE TABLE IF NOT EXISTS dl.orders_daily AS + SELECT *, CAST(NULL AS VARCHAR) AS _wm_partition FROM () WHERE false; -- first-run bootstrap + ALTER TABLE dl.orders_daily SET PARTITIONED BY (_wm_partition); + BEGIN TRANSACTION; + DELETE FROM dl.orders_daily WHERE _wm_partition = '2026-06-19'; + INSERT INTO dl.orders_daily SELECT *, '2026-06-19' AS _wm_partition FROM (); + COMMIT; + ``` + + The strategy variants are all DELETE+INSERT-shaped — no `MERGE INTO`, which + DuckLake can't reliably run on a fresh partition (it 404s writing the first + rows): + - **whole-table replace** (no `// partitioned`) → a single `CREATE OR REPLACE + TABLE … AS ` (handles schema changes, still snapshots). + - **`key=`** → `DELETE FROM … WHERE [ AND] IN (SELECT + FROM ())` then `INSERT` (upsert within the slice). + - **`append`** → the `DELETE` is dropped (insert-only). + +2. **`manual`.** The user writes their own DDL inside their own `BEGIN … COMMIT`; + Windmill injects nothing into the body and only records state (no snapshot + capture, no idempotency guarantee). + +The `_wm_partition` column is the physical link the orchestration layer lacks: on +first materialize Windmill runs `ALTER TABLE … SET PARTITIONED BY (_wm_partition)` +so DuckLake prunes on read and DELETE-by-partition rewrites only that partition's +Parquet files. + +> Storage: DuckLake writes go to `s3://_default_/` through the windmill S3 proxy +> (`/api/w/{ws}/s3_proxy`, gated behind the `parquet` + `private` features). The +> proxy must sign the SigV4 canonical URI with **single** percent-encoding — the +> SigV4 default (`Double`) 401s Hive-partition keys like `_wm_partition=2026-06-19` +> (the `=` double-encodes to `%253D` vs the client's `%3D`). + +### Run summary capture + +After the generated blocks, Windmill appends one read block — it is both the job's +result (a useful preview rendered as the materialized table) and the row it records: + +```sql +SELECT 'ducklake:///
' AS materialized, + '' AS partition, -- only when partitioned + (SELECT count(*) FROM [WHERE _wm_partition = '']) AS rows, + (SELECT max(snapshot_id) FROM ducklake_snapshots('')) AS snapshot_id; +``` + +The `snapshot_id` and `rows` are persisted as `materialized_partition` metadata. +One extra round-trip per materialization, no new infra. + +## Metadata schema + +Extends the `materialized_partitions` table proposed in `pipelines-vs-dbt.md` +§"First implementation slice" with the DuckLake snapshot id: + +``` +materialized_partition ( + workspace_id TEXT, + asset_kind TEXT, -- 'ducklake' + asset_path TEXT, -- 'analytics/orders_daily' + partition TEXT, -- '2026-06-19' (NULL for unpartitioned) + snapshot_id BIGINT, -- DuckLake snapshot produced by this materialize + row_count BIGINT, + job_id UUID, + materialized_at TIMESTAMPTZ, + PRIMARY KEY (workspace_id, asset_kind, asset_path, partition) +) +``` + +This one table drives four things at once: + +- **Observability** — "last materialized: snapshot 42, 1.2M rows, 09:14" per + asset node (closes the Dagster-catalog gap from the v1-readiness review). +- **Run-stale / gap detection** — which partitions exist, which are missing. +- **Backfill** — the missing/failed set *is* the backfill worklist. +- **Snapshot pinning** — see below. + +## Reproducibility — the beyond-dbt part + +Because every materialization records the snapshot it produced, a downstream +consumer can read the *exact* upstream snapshot its run saw: + +```sql +FROM dl.orders_daily AT (VERSION => $WM_UPSTREAM_SNAPSHOT) +``` + +The cascade already threads a `trigger` blob (producer path, partition) to each +subscriber; add the producer's captured `snapshot_id` to it, and a consumer's +read is pinned to the upstream state at dispatch time. That makes the *whole +pipeline* reproducible and time-travelable — something dbt has no native answer +for (dbt models are always "whatever's in the warehouse now"). It also gives +rollback (re-point an asset to snapshot N) and "what did this table look like at +the failing run" debugging, for free off the same captured ids. + +This is the differentiator worth leaning on. It is not catch-up to dbt; it is a +capability dbt structurally cannot offer, and DuckLake gives it to us at the +cost of recording one integer per run. + +It also means **we do not build SCD2 snapshots** (gap #4 in `pipelines-vs-dbt.md`): +DuckLake time-travel is a strictly better answer for most of what dbt's +`{% snapshot %}` is used for. One fewer engine to write. + +## Scoping decision: DuckLake vs DataTable + +**Make DuckLake the materialization/versioning substrate; keep DataTable as the +live operational table with no versioning.** DataTable is plain Postgres +(`transform_attach_datatable`, `duckdb_executor.rs:742`) — no native snapshots +or time-travel — so giving *it* the versioned/incremental story means building +MVCC-on-top ourselves (history tables, SCD2), precisely the complexity this +DuckLake approach exists to avoid. Clean split: + +- `ducklake://` → analytics, versioned, reproducible, backfillable. +- `datatable://` → mutable app/operational state; partition idempotency via + DELETE+INSERT still works, but no snapshot/time-travel layer. + +Don't try to give both the full treatment for v1. + +## v1 slice (smallest viable) + +1. **Partition runtime context** — resolve `(value, start, end)` and surface as + `WM_PARTITION*` bind/env (Path C step 1; partly built per the + pipeline-partition-runtime work). +2. **Physical partition wiring** — `_wm_partition` column + `SET PARTITIONED BY` + on first materialize for `ducklake://` targets. +3. **Strategy templates** — DELETE+INSERT default (`CREATE OR REPLACE` for the + whole table); delete-by-key + insert when `key=`; INSERT-only when + `append`. Managed `// materialize` wraps a single-SELECT DuckDB script behind + these templates; `// materialize manual` opts out. +4. **Snapshot + metadata capture** — append `ducklake_snapshots` read, persist + `materialized_partition` rows. +5. **Surface it** — last-materialized/snapshot/row-count on the asset node; + missing-partition set feeds the backfill UI. +6. *v1.x* — snapshot pinning across the cascade (`$WM_UPSTREAM_SNAPSHOT`), + rollback, time-travel read helper. + +Steps 1–5 are a thin annotation+template layer plus one metadata table and one +extra read per run. They deliver managed/incremental/versioned assets, +idempotent partitioned materialization, the backfill substrate, and +materialization observability together — and stay recognizably Windmill-shaped. + +## Open decisions + +These ride on top of the six in `pipelines-vs-dbt.md` §"Decisions either path +forces"; DuckLake-specific: + +1. **Bootstrap of `SET PARTITIONED BY`.** First-materialize detection — table + absent vs. present-but-unpartitioned. Idempotent re-apply. +2. **Snapshot retention / compaction.** DuckLake snapshots accumulate; when do + we expire old ones, and does pinning hold a snapshot alive past retention? +3. **Pin scope.** Pin only direct producers, or the full transitive upstream + set per run? Storage and "stale pin" semantics differ. +4. **Managed multi-statement.** *Resolved:* managed `// materialize` accepts + setup statements (ATTACH/SET/…) followed by exactly one trailing SELECT, and + rejects anything else at deploy with a clear error pointing to + `// materialize manual`. The classifier (`sql_materialize.rs`) is the single + source of truth. diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md new file mode 100644 index 0000000000..4a957b2a42 --- /dev/null +++ b/docs/pipelines-vs-dbt.md @@ -0,0 +1,379 @@ +# Pipelines vs. dbt + +Positioning analysis and architectural notes for the data-pipeline abstraction +currently landing on `feat/asset-graph-view`. Covers what we're building, how +it differs from dbt, which dbt features are real gaps vs. TODO, and a focused +deep-dive on incremental materialization — including a recommendation to +collapse it into partitioning rather than ship it as a separate concept. + +## What we're building + +Asset-centric, polyglot, annotation-driven, event-aware: + +- Assets (`datatable`, `ducklake`, `s3object`, `volume`) are graph nodes; + scripts are edges that produce/consume them. See + `backend/parsers/windmill-parser/src/asset_parser.rs:25`. +- Lineage comes from two sources: parsed annotations (`// pipeline`, + `// on datatable://...`, `// partitioned daily`, `// freshness 1h`, + `// trigger any`, `// debounce`, `// tag`, `// retry`) and body-inferred + reads/writes via the asset parser. +- Triggers are first-class: schedule, webhook, email, kafka, mqtt, nats, + postgres, sqs, gcp — all wired into the same DAG view + (`frontend/src/lib/components/assets/AssetGraph/types.ts:56`). +- Per-language scaffolds (DuckDB ATTACH, Postgres, Python, TS, Bash) generate + starter code per `PipelineOutputKind` (`datatable | ducklake | s3_parquet | + s3_object | none`). See + `frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts:11`. + +## Differentiators vs. dbt + +1. **Event-driven + batch in one DAG.** dbt is batch-on-warehouse. Kafka → + Python normalize → DuckDB aggregate → Postgres view → Slack notify is + native here; in dbt land it's "use Airflow/Prefect for the non-SQL parts." +2. **Polyglot, not SQL+Jinja.** Python/TS/Bash/Duck/PG transformations live + in the same graph. No Jinja templating language; annotations are real + comments parsed strictly. +3. **Multi-substrate by design.** `datatable` (Postgres), `ducklake` + (lakehouse), `s3_parquet`, `s3_object` are peers. dbt's universe is + "tables in your warehouse." +4. **One platform.** Same runtime as workflows, internal apps, background + jobs, RBAC, secrets, schedules. dbt is single-purpose. +5. **Inferred lineage from code.** Body parser picks up `CREATE TABLE` / S3 + writes — annotations are not strictly required to get edges. dbt requires + explicit `{{ ref() }}` everywhere. + +## Where dbt wins today + +| Gap | Architectural blocker? | Verdict | +|---|---|---| +| Data tests | No | Pure TODO | +| Incremental materializations | No, but pick a philosophy | TODO with design decision | +| Column lineage + docs site | No | Pure TODO | +| Snapshots / SCD2 | No | New output kind | +| Selective execution grammar | No | UI/CLI surface | +| Schema contracts | No, but design metadata model | TODO with design work | +| Packages / community | Closed annotation parser starts to bind | Decide extensibility model | +| Semantic layer / metrics | No | Large additive scope | + +The three items where the current abstraction needs deliberate decisions +before more weight lands on it: **incremental philosophy, schema metadata, +and annotation extensibility**. The rest is execution. + +### 1. Data tests + +dbt: `unique`, `not_null`, `accepted_values`, custom generic tests, plus +singular tests. Run as `SELECT` statements that pass when they return 0 rows. + +Today: nothing. Annotation parser is the natural hook — +`// test unique col_name`, `// test not_null col_name`, +`// test ` for custom. Pipeline runtime already handles +failure propagation. Lowest-risk, highest-payoff item. + +### 2. Incremental materializations + +See [Incremental deep-dive](#incremental-deep-dive) below. + +### 3. Column lineage + docs + +dbt: SQL-AST parsing for column-level deps; `dbt docs serve` produces a +static lineage site with descriptions. + +Today: graph is asset-level. `SqlQueryDetails` in the parser +(`backend/parsers/windmill-parser/src/asset_parser.rs:44`) already has a +column map — the scaffolding exists. No `// column` annotation, no docs +surface. Pure TODO; no abstraction stands in the way. + +### 4. Snapshots / SCD2 + +dbt: `{% snapshot %}` blocks with `strategy='timestamp'` or `'check'`. +Today: nothing. Add as a new `PipelineOutputKind` + `// snapshot strategy= +timestamp updated_at=updated_at unique_key=id` annotation. Same shape as +other output kinds. + +### 5. Selective execution grammar + +dbt: `--select tag:nightly+ state:modified+ +my_model+`. +Today: `requestRunCascadeSignal` in the canvas, `// tag` annotation parsed. +Graph + tags + last-run state has all the inputs. UI/CLI surface, not +abstraction work. + +### 6. Schema contracts + +dbt: `contract: enforced` + `columns: [{name, data_type}]`. Compile-time +check that model output matches the declaration. + +Today: `// on datatable://users/active` is a string. Rename a column +upstream → downstream breaks at runtime, silently. + +This is the item where the current asset abstraction is thinnest. +To do contracts well: capture output schemas after a run (substrate-specific +DESCRIBE), persist them as asset metadata, validate consumer references at +save time. The asset-as-typed-node model accommodates it — but **where** +schemas live (asset row, sidecar?), **when** they're captured (post-run? +edit-time?), and **how** versioning works are non-trivial design choices. +Worth doing intentionally now while the asset surface is still young. + +### 7. Packages / community + +dbt: `dbt deps`, `dbt-utils`, `dbt-expectations`. Whole ecosystem on Jinja +macros. + +Today: closed-vocabulary annotation parser — `parsePipelineAnnotations` +hardcodes `pipeline`, `partitioned`, `freshness`, `trigger`, `debounce`, +`tag`, `retry`, `on`. No way for a package to register +`// test rows_between 100 1000000` or `// hook on_failure my_alert`. + +This is the one place the current abstraction starts to bind. Macros are +also dbt's biggest pain source — we don't have to replicate them. Possible +shapes: + +- **Hooks-as-scripts**: `// on_failure f/lib/alert`, `// pre_run f/lib/setup`. + Value is a script path. Stays inside the closed annotation set; new + hook *types* still require parser changes but third-party *behavior* + ships as scripts. +- **Test types as scripts**: a test is a script that returns 0/1, packaged + via the hub like anything else. +- **Materialization plugins**: harder; template-generator would need to be + extensible. + +Doing this *after* you've shipped 30 hardcoded annotations is much harder +than doing it now. + +### 8. Semantic layer / metrics + +dbt: `metrics:` blocks, MetricFlow, BI-tool query API. Large scope, +additive. Lowest priority of the eight. + +## Incremental deep-dive + +### How dbt incremental works + +```sql +-- models/marts/orders_daily.sql +{{ config( + materialized='incremental', + unique_key='order_id', + incremental_strategy='merge', + on_schema_change='append_new_columns' +) }} + +SELECT order_id, user_id, amount, created_at +FROM {{ ref('orders_raw') }} +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %} +``` + +- **First run** (target doesn't exist): `CREATE TABLE orders_daily AS + SELECT ...` — full build, no WHERE. +- **Subsequent runs**: stage to temp table, then MERGE on `unique_key`. + +Knobs: `incremental_strategy` ∈ {`merge`, `append`, `delete+insert`, +`insert_overwrite`, `microbatch`}. `on_schema_change` ∈ {`fail`, `ignore`, +`append_new_columns`, `sync_all_columns`}. `--full-refresh` forces rebuild. + +Pain points: watermark + unique_key interaction is subtle (late-arriving +rows past the watermark are silently dropped); `on_schema_change` defaults +to `ignore` (silent column drop); Jinja `is_incremental()` runs at compile, +not runtime — debugging requires `dbt compile`; cross-warehouse MERGE +dialect is dbt's biggest internal complexity. + +### Where Windmill stands today + +- `// partitioned daily|hourly|weekly|monthly|dynamic key=...` parsed into + `PartitionSpec` at + `backend/parsers/windmill-parser/src/asset_parser.rs:172`. +- `// freshness 1h` parsed. +- Templates emit `CREATE TABLE IF NOT EXISTS ... AS SELECT *` — full + refresh, every run, no partition substitution. +- No `WM_PARTITION_*` context flowing into scripts. +- No materialized-partition state per asset. + +Annotations are present but metadata-only. Nothing actually executes +incrementally yet. + +### Path A — Literal templates ("script is the truth") + +Philosophy: WYSIWYG. Windmill never wraps. Templates scaffold boilerplate, +partition context is injected as bind / env vars, the user owns the SQL. + +```sql +-- pipeline +-- on datatable://prod/orders_raw +-- partitioned daily +-- unique_key order_id + +ATTACH 'datatable://prod' AS pg; + +CREATE TABLE IF NOT EXISTS pg.orders_daily ( + order_id BIGINT PRIMARY KEY, + user_id BIGINT, + amount NUMERIC, + created_at TIMESTAMPTZ +); + +CREATE OR REPLACE TEMP TABLE _stage AS +SELECT order_id, user_id, amount, created_at +FROM pg.orders_raw +WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END; + +BEGIN; +DELETE FROM pg.orders_daily + WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END; +INSERT INTO pg.orders_daily SELECT * FROM _stage; +COMMIT; +``` + +Runtime: resolve `(value, start, end)` from scheduler tick / trigger event +/ backfill range → bind as SQL params → execute script as-is → record +`(asset_path, partition_value)` on success. + +**Pros**: no compile step; backfill is trivial (idempotent DELETE+INSERT); +late-arriving data → just re-run the affected partition; no dialect +rewriting in core; Python/TS/Bash/SQL all fit the same model. + +**Cons**: boilerplate per script; materialization changes require script +edits; user owns dialect specifics. + +### Path B — dbt-style wrapping + +Philosophy: separate intent (SELECT) from execution (DDL). User declares +what; Windmill compiles to per-substrate DDL. + +User writes: + +```sql +SELECT order_id, user_id, amount, created_at +FROM pg.orders_raw +WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END +``` + +Runtime parses, looks up target schema, wraps per output kind + +strategy + first-run/subsequent-run state. + +**Pros**: concise; materialization is a config flip; automatic schema-drift +handling; cross-substrate consistency. + +**Cons**: two-layer execution ("what ran?" needs a compile-output view); +SELECT-only restricts pre/post-statement work (dbt's answer: `pre_hook` / +`post_hook` — more surface); doesn't generalize to Python/TS (you end up +with two execution models); cross-substrate MERGE dialect is where dbt has +burned the most engineering — we'd inherit that tax forever; schema +introspection per substrate is its own project. + +### Path C — Hybrid (recommended) + +- **Literal-by-default**: scaffolds emit full DDL with `WM_PARTITION_*` + substitution. WYSIWYG for all languages. +- **Helper library** (e.g. `wmll.partition`, `wmll.datatable.upsert_partition`): + lifts boilerplate into library calls without hiding semantics — readable + source. +- **Opt-in wrapping** for single-SELECT SQL scripts via + `// materialized incremental wrap=true`. Limit to DuckDB first; add + others as needed. Always log the compiled SQL. +- **State + backfill UI**: persist materialized partitions per asset; UI + to backfill a range with concurrency cap. + +Ships A's 80% case first without committing to B's dialect-rewriting tax. +Wrapping becomes opt-in convenience for users who want dbt-style ergonomics. + +> See [`ducklake-materialization.md`](./ducklake-materialization.md) for the +> DuckLake-native realization of this path: how snapshots make the assets +> versioned/reproducible for free, the executor codegen seam, and the +> materialization-metadata schema. + +### Decisions either path forces + +1. **Partition window provenance.** Scheduler tick? Trigger event time + (Kafka `event_time` header)? Explicit backfill? Default = "now's bucket"? +2. **Surface.** Bind params (`$WM_PARTITION_START`), env vars + (`WM_PARTITION_START`), helper library — probably all three for + different languages, but pick canonical names. +3. **First-run bootstrap.** Template scaffolds `CREATE TABLE IF NOT + EXISTS` (A), or runtime detects "table missing → full refresh" (B). +4. **State tracking.** `materialized_partitions` keyed by `(workspace, + asset_kind, asset_path)`. Drives "run stale," backfill gap detection, + downstream waiting. +5. **Backfill execution.** N partitions → serial? Parallel with + concurrency cap per asset? +6. **Idempotency contract.** `// partitioned` should imply "re-running the + same partition is safe." Templates and helpers must enforce. + +## Partitioning vs. incremental: the reframing + +Partitioning covers ~80% of what dbt's incremental does. What it gives +for free: + +- Unit of work (one partition per run) +- Idempotency (DELETE-by-partition + INSERT is safe to rerun) +- State (track which partitions are materialized) +- Backfill (re-run a range) +- First-run vs. subsequent-run (every run is "process partition P" — no + special case) +- "Process only new data" (the partition window IS the filter) + +dbt itself has been migrating toward partition-first thinking via +`microbatch` strategy — essentially `incremental` with mandatory partition +key. + +### What partitioning alone doesn't address + +**Dedup within a partition by a separate key.** Example: partition by +`created_at` daily, but `orders_raw` is mutable — the same `order_id` can +appear multiple times in one partition (initial create, then amendments). +You want `orders_daily` to hold the latest version per `order_id`. + +DELETE-by-partition + INSERT works only if you reprocess from a +source-of-truth source. If you're consuming amendments and need dedup +*within* the slice, you need MERGE on `order_id`, not DELETE on partition. + +This is what dbt's `unique_key` does. Orthogonal to partitioning: +`partitioned` answers "which slice?"; `unique_key` answers "how do I dedup +inside the slice?" + +**Pure watermark-based incremental.** Mostly subsumed by +`// partitioned dynamic key=updated_at` — a partition becomes "everything +since the last seen value of `key`." + +### Recommended annotation shape + +Don't build "incremental" as a concept. Build: + +- `// partitioned ` — unit of work + state + backfill (already exists). +- `// unique_key ` — opt-in dedup-within-partition. Drives MERGE + template vs. DELETE+INSERT template. +- `// append` — opt-out of dedup entirely (INSERT-only, no DELETE). + +This collapses dbt's `materialized=incremental` + `incremental_strategy` + +`unique_key` into orthogonal annotations that compose. Partition-first is +the better mental model. + +Schema drift handling (`on_schema_change`) is genuinely separate — applies +to full-refresh too — and belongs with the schema-contracts work (gap #6). + +## First implementation slice + +Sequencing if we go with the hybrid + partition-first reframing: + +1. **Partition runtime context** — resolve `(value, start, end)` from + scheduler / trigger / backfill, surface as bind vars + env vars. No + materialization change yet. +2. **Helper library** — `wmll.partition.window()`, + `wmll.datatable.upsert_partition()` for Python/TS, SQL macros for + DuckDB/PG. +3. **Template updates** — when `// partitioned X` is present, scaffold + DELETE+INSERT (or MERGE when `// unique_key` also present, or INSERT + when `// append`). +4. **Materialized-partition state** — new table keyed by + `(workspace, asset_kind, asset_path, partition_value)`. Asset metadata + read API exposes it. +5. **Backfill UI** — date range picker on the pipeline folder page; fans + out runs with concurrency cap. +6. *Later, behind a flag:* opt-in wrap mode for single-SELECT DuckDB. + +Delivers dbt's pragmatic value (incremental, backfill, idempotent reruns) +without buying the compile-layer maintenance, and keeps Windmill +recognizably Windmill-shaped. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 54b326cff7..2d74ecc1fc 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.727.0", + "version": "1.734.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.727.0", + "version": "1.734.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -78,7 +78,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.673.0", + "windmill-parser-wasm-asset": "1.728.1", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -166,11 +166,6 @@ "svelte": "^5.0.0" } }, - "../backend/parsers/windmill-parser-wasm/pkg-ts": { - "name": "windmill-parser-wasm-ts", - "version": "1.673.0", - "extraneous": true - }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -13667,9 +13662,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.673.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.673.0.tgz", - "integrity": "sha512-sJZ9YbKhxMT67wWilmT7CDbwX0eVr+rvc6Y4OdHzOXwTwu3MixUinzVyP/lD4WDQW6xzitqOfstHBUrOxpww3A==" + "version": "1.728.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.728.1.tgz", + "integrity": "sha512-73cyU6XM3gYEjFBx3qOKnv+VV1t70eAr6OiT+x0QobjFVNmqZFEdA7ayMYYVCnnix8OZxcsTNEF1hT61w1XiKw==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", diff --git a/frontend/package.json b/frontend/package.json index f750d6a2ad..f39b100305 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.727.0", + "version": "1.734.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -153,7 +153,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.673.0", + "windmill-parser-wasm-asset": "1.728.1", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", diff --git a/frontend/src/lib/attachments/arrowTabNav.ts b/frontend/src/lib/attachments/arrowTabNav.ts new file mode 100644 index 0000000000..e98b7db34c --- /dev/null +++ b/frontend/src/lib/attachments/arrowTabNav.ts @@ -0,0 +1,59 @@ +import { tabbable } from 'tabbable' +import type { Attachment } from 'svelte/attachments' + +export interface ArrowTabNavOptions { + /** Which arrow-key pair walks the tab order. Default `'y'` (Up/Down). */ + axis?: 'x' | 'y' + /** Use this to layer custom keys (Enter, Escape …) */ + onKeyDown?: (e: KeyboardEvent) => void +} + +/** + * Map one axis of arrow keys to next/previous in Tab order, scoped to + * the attached element. Wraps around at the ends. Bails when the + * keypress originates inside a text input / textarea / contenteditable + * so the caret can still move with the arrows. + * + *
…
+ *
…
+ *
{ … } })}>…
+ */ +export function arrowTabNav(opts: ArrowTabNavOptions = {}): Attachment { + const axis = opts.axis ?? 'y' + const nextKey = axis === 'y' ? 'ArrowDown' : 'ArrowRight' + const prevKey = axis === 'y' ? 'ArrowUp' : 'ArrowLeft' + + return (node) => { + const handler = (e: KeyboardEvent) => { + // Run the consumer's handler first so they can preventDefault + // or do their own thing before we react to arrows. + opts.onKeyDown?.(e) + + if (e.key !== nextKey && e.key !== prevKey) return + + // Let editable fields keep their native caret behavior. + const t = e.target as HTMLElement | null + if (t && (t.tagName === 'INPUT' || t.tagName === 'TEXTAREA' || t.isContentEditable)) { + return + } + + const items = tabbable(node) + if (items.length === 0) return + + const active = document.activeElement as HTMLElement | null + const i = active ? items.indexOf(active) : -1 + const dir = e.key === nextKey ? 1 : -1 + // No tabbable currently focused: jump to the first (next) or last (prev). + const next = + i === -1 + ? items[dir === 1 ? 0 : items.length - 1] + : items[(i + dir + items.length) % items.length] + + e.preventDefault() + next?.focus() + } + + node.addEventListener('keydown', handler) + return () => node.removeEventListener('keydown', handler) + } +} diff --git a/frontend/src/lib/attachments/selectAndAdvanceTo.ts b/frontend/src/lib/attachments/selectAndAdvanceTo.ts new file mode 100644 index 0000000000..0de23572c5 --- /dev/null +++ b/frontend/src/lib/attachments/selectAndAdvanceTo.ts @@ -0,0 +1,49 @@ +import { tabbable } from 'tabbable' + +/** + * Build an Enter-key handler for `arrowTabNav`'s `onKeyDown` (or any + * `keydown` listener): activate whatever is focused (so its `onClick` + * fires), then move focus to the first tabbable inside the container + * returned by `getNext`. preventDefault suppresses the browser's own + * Enter→click so we don't double-fire. + * + * By default the handler bails inside `