diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 1284154a91..580a757c21 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -15,8 +15,8 @@ on: description: "Space-separated modes (global, cli, flow, app, script)" default: "global cli" models: - description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones" - default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" + description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for" + default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -28,7 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. @@ -40,17 +40,28 @@ jobs: outputs: matrix: ${{ steps.matrix.outputs.matrix }} steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: ai_evals/core + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.0 + - id: matrix env: MODES: ${{ inputs.modes || 'global cli' }} - # cli drives the Claude Agent SDK, so it only takes the Claude aliases - # (`cli` entries in ai_evals/core/models.ts). + # cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts. run: | - echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" ' + cli_models=$(bun -e ' + import { EVAL_MODELS } from "./ai_evals/core/models.ts"; + console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli) + .flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));') + echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" ' def words: split(" ") | map(select(. != "")); [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode | $models | words[] - | select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i")) + | select($mode != "cli" or (ascii_downcase | IN($cli[]))) | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT" evals: @@ -120,6 +131,10 @@ jobs: working-directory: ./cli run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh + - name: Install Codex CLI + if: matrix.mode == 'cli' + run: npm install --global @openai/codex@0.159.3 + - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }} working-directory: ./ai_evals env: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3f035f45c0..cc72934156 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.826.0" + ".": "1.828.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce5ffcde3..c958457c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,48 @@ # Changelog +## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08) + + +### Features + +* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864)) +* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d)) +* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4)) +* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6)) +* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24)) + + +### Bug Fixes + +* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775)) +* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba)) +* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687)) + + +### Performance Improvements + +* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996)) + +## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07) + + +### Features + +* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9)) +* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842)) +* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8)) +* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f)) +* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971)) + + +### Bug Fixes + +* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3)) +* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67)) +* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2)) +* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1)) +* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7)) + ## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07) diff --git a/ai_evals/README.md b/ai_evals/README.md index bff4294e52..44870ba2a2 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -89,7 +89,21 @@ Notes: - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5` - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases -- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there +- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there: + + | Agent | Models | Label | Needs | + | --- | --- | --- | --- | + | Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:` | `ANTHROPIC_API_KEY` | + | Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:` | `DEEPSEEK_API_KEY` | + | Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` | + + Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no + Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace + records any command that touches `.agents/skills//SKILL.md` as a `Skill` call to ``, just + before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the + `invokes skill … before first mutation` checks compare the same things for both agents. Codex has no + turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a + Claude Code run does at its cap. - the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs ## Case Format @@ -164,6 +178,12 @@ the decrypted value, exactly as against a real backend. The chat's read path pas Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and assert it via `valueExcludes` to catch a leak. +`runtime.userAnswers` scripts the user's side of `askUserQuestion` in a global case: one +answer per question, in the order they are asked, and a question past the last answer is +dismissed, so `userAnswers: []` dismisses every question. Without it the tool reports that +it cannot ask, and the model usually carries on with its own guess, which makes "asks +before creating anything" impossible to assert. + `toolExpect.toolCallArgs` entries support `sharedByAtLeast: `: at least `n` recorded calls to that tool must carry the same non-blank string in the field. Use it for calls that have to share an identifier, like two test runs of one chat conversation. diff --git a/ai_evals/adapters/cli/codex.test.ts b/ai_evals/adapters/cli/codex.test.ts new file mode 100644 index 0000000000..fdeda1d26b --- /dev/null +++ b/ai_evals/adapters/cli/codex.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "bun:test"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex"; + +describe("unwrapShellCommand", () => { + it("strips the login-shell wrapper Codex reports", () => { + expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe( + "cat .agents/skills/x/SKILL.md" + ); + expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe( + `printf 'hi' > "a b"` + ); + expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`); + expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push"); + expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`); + }); +}); + +describe("toToolInvocations", () => { + it("records reading a SKILL.md as a Skill call before the command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`, + }); + expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([ + ["Skill", { skill: "write-script-bun" }], + ["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }], + ]); + }); + + it("records a skill read after a mutation earlier in the same command", () => { + const tools = toToolInvocations({ + type: "command_execution", + command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`, + }); + expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]); + }); + + it("maps a patch to Edit so it counts as a mutation", () => { + expect( + toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0] + ).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } }); + }); +}); + +describe("runCodex", () => { + it("keeps parallel tool calls in the order they started", async () => { + const dir = await mkdtemp(join(tmpdir(), "fake-codex-")); + const events = [ + { type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } }, + { type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } }, + { type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } }, + { type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } }, + ]; + const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n"); + await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`); + await chmod(join(dir, "codex"), 0o755); + try { + const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` }); + expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]); + expect(result.output).toBe("done"); + expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/ai_evals/adapters/cli/codex.ts b/ai_evals/adapters/cli/codex.ts new file mode 100644 index 0000000000..b49c441556 --- /dev/null +++ b/ai_evals/adapters/cli/codex.ts @@ -0,0 +1,241 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types"; +import { isLikelyMutatingBashCommand } from "./mutation"; + +export interface AgentRunResult { + output: string; + toolsUsed: CliToolInvocation[]; + assistantMessageCount: number; + tokenUsage: BenchmarkTokenUsage | null; + finalContextTokens: number | null; +} + +interface CodexItem { + type?: string; + text?: string; + command?: string; + changes?: { path?: string; kind?: string }[]; + [key: string]: unknown; +} + +interface CodexEvent { + type?: string; + item?: CodexItem; + usage?: { input_tokens?: number; output_tokens?: number }; + message?: string; + error?: { message?: string }; +} + +const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g; + +// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary +// the validators read: shell commands become `Bash`, patches `Edit`, and +// reading `.agents/skills//SKILL.md` (how Codex loads a skill) becomes a +// `Skill` call recorded just before that command. +export async function runCodex( + prompt: string, + cwd: string, + maxTurns: number, + model: string, + env: Record +): Promise { + // A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run. + const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-")); + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const messages: string[] = []; + let failure: string | null = null; + let lastError: string | null = null; + let stderr = ""; + + try { + const child = spawn( + "codex", + [ + "exec", + "--json", + "--ephemeral", + "--skip-git-repo-check", + "--dangerously-bypass-approvals-and-sandbox", + "--model", + model, + "--cd", + cwd, + "-", + ], + { + cwd, + env: { + ...env, + HOME: home, + CODEX_HOME: home, + CODEX_API_KEY: env.OPENAI_API_KEY ?? "", + }, + stdio: ["pipe", "pipe", "pipe"], + } + ); + // Without this, a missing `codex` binary leaves the stdout loop waiting forever. + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.stdin.on("error", () => {}); + child.stdin.end(prompt); + child.stderr.on("data", (chunk) => { + stderr = (stderr + chunk).slice(-4000); + }); + const exited = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", resolve); + }); + + // Codex has no turn cap. A turn starts when the agent issues tool calls with + // none in flight, so parallel calls count once, as they do for Claude Code. + // Claude Code's cap includes the final reply, so the maxTurns-th tool round is + // already over it. + // ponytail: parallel calls that finish before the next one starts count as + // separate turns; count model calls if Codex ever reports them. + let turns = 0; + let inFlight = 0; + let capped = false; + // Parallel calls complete out of order; the trace keeps the order they started in. + const slots = new Map(); + const ordered: CliToolInvocation[][] = []; + const slotFor = (item: CodexItem) => { + const id = typeof item.id === "string" ? item.id : undefined; + if (id !== undefined && slots.has(id)) { + return slots.get(id)!; + } + ordered.push([]); + if (id !== undefined) { + slots.set(id, ordered.length - 1); + } + return ordered.length - 1; + }; + + for await (const line of createInterface({ input: child.stdout })) { + const event = parseEvent(line); + if (!event) { + continue; + } + const item = event.item; + + if (event.type === "item.started" && item && isToolItem(item)) { + if (inFlight === 0) { + turns += 1; + } + inFlight += 1; + slotFor(item); + if (turns >= maxTurns && !capped) { + capped = true; + child.kill("SIGTERM"); + } + } else if (event.type === "item.completed" && item) { + if (item.type === "agent_message") { + result.assistantMessageCount += 1; + if (typeof item.text === "string") { + messages.push(item.text); + } + } else if (isToolItem(item)) { + inFlight = Math.max(0, inFlight - 1); + ordered[slotFor(item)] = toToolInvocations(item); + } + } else if (event.type === "turn.completed" && event.usage) { + const prompt = event.usage.input_tokens ?? 0; + const completion = event.usage.output_tokens ?? 0; + result.tokenUsage = { prompt, completion, total: prompt + completion }; + } else if (event.type === "turn.failed") { + failure = event.error?.message ?? JSON.stringify(event); + } else if (event.type === "error") { + // Also sent for retries Codex recovers from ("Reconnecting... 1/5"). + lastError = event.message ?? JSON.stringify(event); + } + } + + result.toolsUsed = ordered.flat(); + const code = await exited; + if (failure) { + throw new Error(`codex: ${failure}`); + } + // Fail the attempt, as Claude Code does at its cap. + if (capped) { + throw new Error(`codex: reached maximum number of turns (${maxTurns})`); + } + if (code !== 0) { + throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`); + } + } finally { + await rm(home, { recursive: true, force: true }); + } + + result.output = messages.join("\n"); + return result; +} + +export function toToolInvocations(item: CodexItem): CliToolInvocation[] { + const timestamp = Date.now(); + + if (item.type === "command_execution" && typeof item.command === "string") { + const command = unwrapShellCommand(item.command); + // A skill read counts as preceding the command only when nothing before it in + // the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a + // skill-before-first-mutation check. + const before: string[] = []; + const after: string[] = []; + for (const match of command.matchAll(SKILL_FILE_PATTERN)) { + const skill = match[1]!; + if (before.includes(skill) || after.includes(skill)) { + continue; + } + (isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill); + } + const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp }); + return [ + ...before.map(skillCall), + { tool: "Bash", input: { command }, timestamp }, + ...after.map(skillCall), + ]; + } + + if (item.type === "file_change") { + return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }]; + } + + const { id: _id, type, status: _status, ...input } = item; + return [{ tool: String(type), input, timestamp }]; +} + +// Codex reports commands as run through the user's login shell, e.g. +// `/usr/bin/zsh -lc 'cat SKILL.md'`. +export function unwrapShellCommand(command: string): string { + const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/); + if (!match) { + return command; + } + const body = match[2]!; + return match[1] === "'" + ? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'") + : body.replace(/\\(["\\$`])/g, "$1"); +} + +function isToolItem(item: CodexItem): boolean { + return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? ""); +} + +function parseEvent(line: string): CodexEvent | null { + try { + const value = JSON.parse(line); + return value && typeof value === "object" ? (value as CodexEvent) : null; + } catch { + return null; + } +} diff --git a/ai_evals/adapters/cli/mutation.ts b/ai_evals/adapters/cli/mutation.ts new file mode 100644 index 0000000000..05b29d68d7 --- /dev/null +++ b/ai_evals/adapters/cli/mutation.ts @@ -0,0 +1,10 @@ +export function isLikelyMutatingBashCommand(command: string): boolean { + return ( + /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || + /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || + /\bsed\s+-i\b/.test(command) || + /\bperl\s+-pi\b/.test(command) || + // In command position only: `cat AGENTS.wmill.md` is a read. + /(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command) + ); +} diff --git a/ai_evals/adapters/cli/runtime.test.ts b/ai_evals/adapters/cli/runtime.test.ts index 3ea53ca587..d3e537b659 100644 --- a/ai_evals/adapters/cli/runtime.test.ts +++ b/ai_evals/adapters/cli/runtime.test.ts @@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test"; import { anthropicUsageToBenchmarkTokenUsage, extractCliResultTokenUsage, + endpointEnv, extractProposedWmillCommands, + getFirstMutationToolIndex, parseWmillInvocationLog, } from "./runtime"; @@ -147,3 +149,32 @@ push ]); }); }); + +describe("getFirstMutationToolIndex", () => { + it("counts wmill only as a command, not in a file name", () => { + const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 }); + expect( + getFirstMutationToolIndex([ + bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"), + bash("ls && wmill generate-metadata"), + ]) + ).toBe(1); + expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0); + }); +}); + +describe("endpointEnv", () => { + it("passes the endpoint only its own key", () => { + const env = endpointEnv( + { baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" }, + "m", + { PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" } + ); + expect(env).toEqual({ + PATH: "/bin", + ANTHROPIC_BASE_URL: "https://example.test", + ANTHROPIC_API_KEY: "ds", + ANTHROPIC_DEFAULT_HAIKU_MODEL: "m", + }); + }); +}); diff --git a/ai_evals/adapters/cli/runtime.ts b/ai_evals/adapters/cli/runtime.ts index b45df70011..e7368db366 100644 --- a/ai_evals/adapters/cli/runtime.ts +++ b/ai_evals/adapters/cli/runtime.ts @@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk"; import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { delimiter, join } from "path"; import { fileURLToPath } from "url"; -import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models"; +import { + getCliEvalModel, + resolveEvalModel, + type CliEvalModelConfig, +} from "../../core/models"; +import { runCodex, type AgentRunResult } from "./codex"; +import { isLikelyMutatingBashCommand } from "./mutation"; +export { formatCliRunModelLabel } from "../../core/models"; import type { BenchmarkTokenUsage, CliToolInvocation, @@ -141,16 +148,94 @@ export async function runPromptAndCapture( maxTurns: number = 3, modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL ): Promise { - const toolsUsed: ToolInvocation[] = []; - const skillsInvoked: string[] = []; - const bashCommands: string[] = []; - let output = ""; - let assistantMessageCount = 0; - let tokenUsage: BenchmarkTokenUsage | null = null; - let finalContextTokens: number | null = null; const startedAt = Date.now(); const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME); const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME); + const env = { + ...getQueryEnv(), + PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, + WMILL_BENCHMARK_LOG_PATH: wmillLogPath, + }; + + await installWmillStub(stubBinDir); + + const run = + modelConfig.runtime === "codex" + ? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env) + : await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env); + + const skillsInvoked: string[] = []; + const bashCommands: string[] = []; + for (const tool of run.toolsUsed) { + if (tool.tool === "Skill" && typeof tool.input.skill === "string") { + pushUnique(skillsInvoked, tool.input.skill); + } + if (tool.tool === "Bash") { + for (const command of extractBashCommands(tool.input)) { + pushUnique(bashCommands, command); + } + } + } + + const wmillInvocations = await readWmillInvocationLog(wmillLogPath); + + return { + output: run.output, + durationMs: Date.now() - startedAt, + tokenUsage: run.tokenUsage, + finalContextTokens: run.finalContextTokens, + trace: { + toolsUsed: run.toolsUsed, + skillsInvoked, + assistantMessageCount: run.assistantMessageCount, + bashCommands, + proposedCommands: extractProposedWmillCommands(run.output), + executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), + wmillInvocations, + firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed), + }, + }; +} + +// Only the endpoint's own key may reach it: a missing key would make Claude Code +// fall back to the user's Claude login, and the agent's Bash inherits this env, +// so any other credential could end up in a tool result sent to the endpoint. +export function endpointEnv( + endpoint: NonNullable, + model: string, + env: Record +): Record { + const key = env[endpoint.apiKeyEnv]; + if (!key) { + throw new Error(`${endpoint.apiKeyEnv} is not set`); + } + const rest = Object.fromEntries( + Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name)) + ); + return { + ...rest, + ANTHROPIC_BASE_URL: endpoint.baseUrl, + ANTHROPIC_API_KEY: key, + // Claude Code's background calls default to Haiku, which the endpoint does not serve. + ANTHROPIC_DEFAULT_HAIKU_MODEL: model, + }; +} + +async function runClaudeCode( + prompt: string, + cwd: string, + maxTurns: number, + modelConfig: CliEvalModelConfig, + env: Record +): Promise { + const result: AgentRunResult = { + output: "", + toolsUsed: [], + assistantMessageCount: 0, + tokenUsage: null, + finalContextTokens: null, + }; + const endpoint = modelConfig.anthropicEndpoint; const options: Options = { cwd, @@ -158,80 +243,42 @@ export async function runPromptAndCapture( maxTurns, settingSources: ["project"], allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"], - env: { - ...getQueryEnv(), - PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir, - WMILL_BENCHMARK_LOG_PATH: wmillLogPath, - }, + env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env, }; - await installWmillStub(stubBinDir); - for await (const message of query({ prompt, options })) { if (message.type === "assistant") { - assistantMessageCount += 1; + result.assistantMessageCount += 1; const turnContext = anthropicUsageToBenchmarkTokenUsage( message.message?.usage )?.prompt; if (turnContext && turnContext > 0) { - finalContextTokens = turnContext; + result.finalContextTokens = turnContext; } const content = message.message?.content; if (Array.isArray(content)) { for (const block of content) { if (block.type === "tool_use") { - const input = normalizeToolInput(block.input); - toolsUsed.push({ + result.toolsUsed.push({ tool: block.name, - input, + input: normalizeToolInput(block.input), timestamp: Date.now() }); - - if (block.name === "Skill") { - const skillInput = input as { skill?: string }; - if (skillInput.skill) { - pushUnique(skillsInvoked, skillInput.skill); - } - } - - if (block.name === "Bash") { - for (const command of extractBashCommands(input)) { - pushUnique(bashCommands, command); - } - } } else if (block.type === "text") { - output += block.text; + result.output += block.text; } } } } else if (message.type === "result") { const resultMessage = message as { result?: string }; - tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage; + result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage; if (typeof resultMessage.result === "string") { - output += resultMessage.result; + result.output += resultMessage.result; } } } - const proposedCommands = extractProposedWmillCommands(output); - const wmillInvocations = await readWmillInvocationLog(wmillLogPath); - - return { - output, - durationMs: Date.now() - startedAt, - tokenUsage, - finalContextTokens, - trace: { - toolsUsed, - skillsInvoked, - assistantMessageCount, - bashCommands, - proposedCommands, - executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand), - wmillInvocations, - firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed), - }, - }; + return result; } export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean { @@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean return result.trace.toolsUsed.some((tool) => tool.tool === toolName); } -export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string { - return `${modelConfig.provider}:${modelConfig.model}`; -} - export function getToolInputs( result: PromptRunResult, toolName: string @@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string { return ["wmill", ...entry.argv].join(" ").trim(); } -function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { +export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { for (const [index, tool] of toolsUsed.entries()) { if (tool.tool === "Write" || tool.tool === "Edit") { return index; @@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null { return null; } -function isLikelyMutatingBashCommand(command: string): boolean { - return ( - /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) || - /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) || - /\bsed\s+-i\b/.test(command) || - /\bperl\s+-pi\b/.test(command) || - /\bwmill\b/.test(command) - ); -} - function pushUnique(values: string[], value: string): void { if (!values.includes(value)) { values.push(value); diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index be8c1d74cc..fe20d77d05 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -123,6 +123,7 @@ export interface GlobalEvalOptions { planMode?: boolean; model?: string; maxIterations?: number; + userAnswers?: string[]; provider?: AIProvider; backend: WindmillBackendSettings; workspaceRoot?: string; @@ -220,6 +221,7 @@ export async function runGlobalEval( onToolCall: options.runContext?.onToolCall, options: { maxIterations: options.maxIterations, + userAnswers: options.userAnswers, model, workspace: workspaceRoot, provider: options.provider, diff --git a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts index 3d0c6d6734..13aaa59806 100644 --- a/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts @@ -85,6 +85,7 @@ export async function runEval( const model = options.model ?? "gpt-4o"; const maxIterations = options.maxIterations ?? 20; + const pendingAnswers = options.userAnswers && [...options.userAnswers]; const workspace = options.workspace ?? "test-workspace"; const provider = toFrontendEvalProvider(options.provider); @@ -138,6 +139,12 @@ export async function runEval( // edit the arguments, so a case can assert what the model proposed but never how // it reacts to the user changing something. requestRunArgs: async (_toolId, form) => form.args, + ...(pendingAnswers && { + requestUserQuestion: async () => { + const answer = pendingAnswers.shift(); + return answer === undefined ? undefined : [answer]; + }, + }), onNewToken: (token: string) => { if (shouldEmitMessageStart) { onAssistantMessageStart?.(); diff --git a/ai_evals/adapters/frontend/core/shared/types.ts b/ai_evals/adapters/frontend/core/shared/types.ts index c7af6c43e7..1ff783e258 100644 --- a/ai_evals/adapters/frontend/core/shared/types.ts +++ b/ai_evals/adapters/frontend/core/shared/types.ts @@ -16,6 +16,9 @@ export interface ToolCallDetail { export interface EvalRunnerOptions { backend: WindmillBackendSettings; maxIterations?: number; + // Scripted answers to the model's questions, one per question in order; a question past + // the last one is dismissed. Unset, the run cannot be asked anything. + userAnswers?: string[]; model?: string; workspace?: string; provider?: AIProvider; diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 476218a066..b66af3e8dd 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -198,22 +198,127 @@ - uses the requested placeholder value and description - does not create a resource or deploy anything +# A new app that has to store data, with nothing said about where: the data setup is the +# user's call, so it is asked before anything is created. The question is dismissed here, +# which leaves nothing the model may build on. - id: global-test7-ambiguous-app-asks-question prompt: |- Create a new raw app for triaging support tickets. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json runtime: - maxTurns: 4 + maxTurns: 6 + userAnswers: [] validate: draftCountExactly: 0 toolExpect: requiredToolsUsed: + - list_datatables - askUserQuestion forbiddenToolsUsed: - init_app + - exec_datatable_sql - write_app_file - write_app_runnable - deploy_workspace_item - delete_workspace_item + toolCallArgs: + - tool: askUserQuestion + field: question + stringIncludesAnyOf: + - data table + - datatable + - schema + - tables + - store + skipJudge: true + +# The same request with the question answered: the app is built on what the user chose. +- id: global-new-app-data-setup-answer-applied + prompt: |- + Create a new raw app for triaging support tickets. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 32 + userAnswers: + - A new schema named helpdesk + - Create new tables + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - helpdesk + - react-dom/client + toolExpect: + requiredToolsUsed: + - askUserQuestion + - init_app + forbiddenToolsUsed: + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: framework + stringEqualsAnyOf: + - react19 + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - helpdesk + skipJudge: true + +# No framework named, no app to follow, nothing to store: the modal's recommended +# framework is used without a question. +- id: global-new-app-defaults-to-react19 + prompt: |- + Build me a little app that converts temperatures between Celsius and Fahrenheit. + runtime: + maxTurns: 12 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - react-dom/client + toolExpect: + requiredToolsUsed: + - init_app + forbiddenToolsUsed: + - askUserQuestion + - exec_datatable_sql + - deploy_workspace_item + toolCallArgs: + - tool: init_app + field: framework + stringEqualsAnyOf: + - react19 + skipJudge: true + +# The user said where the data goes, so there is nothing left to ask. +- id: global-new-app-storage-specified-no-question + prompt: |- + Build a todo list app in React. Keep the todos in the main data table, in a new + schema called todo_app. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json + runtime: + maxTurns: 20 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: app + valueIncludes: + - todo_app + toolExpect: + requiredToolsUsed: + - init_app + - write_app_runnable + forbiddenToolsUsed: + - askUserQuestion + - deploy_workspace_item + toolCallArgs: + - tool: exec_datatable_sql + field: sql + stringIncludesAnyOf: + - todo_app skipJudge: true - id: global-test8-human-script-infer-path-language diff --git a/ai_evals/core/cases.test.ts b/ai_evals/core/cases.test.ts index 5d6e3245db..b01d4143ba 100644 --- a/ai_evals/core/cases.test.ts +++ b/ai_evals/core/cases.test.ts @@ -192,8 +192,6 @@ describe("loadCases", () => { requiredSkills: ["write-script-bun"], requiredSkillsBeforeFirstMutation: ["write-script-bun"], forbiddenSkills: ["write-script-python3", "write-flow"], - orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"], - orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"], forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"], }); }); diff --git a/ai_evals/core/models.test.ts b/ai_evals/core/models.test.ts index d51d61d224..96f16778d0 100644 --- a/ai_evals/core/models.test.ts +++ b/ai_evals/core/models.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { resolveEvalModel } from "./models"; +import { formatRunModelLabel, resolveEvalModel } from "./models"; describe("resolveEvalModel", () => { it("supports GPT-5.5 aliases for frontend evals", () => { @@ -47,6 +47,18 @@ describe("resolveEvalModel", () => { }); }); + it("labels cli runs by agent, keeping Claude's historical label", () => { + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe( + "anthropic:claude-sonnet-5-5", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe( + "codex:gpt-6.1-sol", + ); + expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe( + "claude-code:deepseek-v4-pro", + ); + }); + it("rejects Gemini aliases for cli evals", () => { expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow( "Model gemini-3-flash-preview is not supported for cli mode", diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 06689c36e1..cf131337fd 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig { model: string; } +// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts). export interface CliEvalModelConfig { - provider: "anthropic"; + runtime: "claude-code" | "codex"; model: string; + // An Anthropic-compatible endpoint other than Anthropic's, for claude-code, + // and the env var holding its key. + anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string }; } export interface EvalModelSpec { @@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-haiku-4-5-20251001", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "haiku", }, }, @@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-4-5-20250929", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "sonnet", }, }, @@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-4-6", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "opus", }, }, @@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-sonnet-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-sonnet-5-5", }, }, @@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "claude-opus-5-5", }, cli: { - provider: "anthropic", + runtime: "claude-code", model: "claude-opus-5-5", }, }, @@ -139,6 +143,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "openai", model: "gpt-6-astra", }, + cli: { + runtime: "codex", + model: "gpt-6-astra", + }, + }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, }, { id: "gpt-6-sol", @@ -149,6 +166,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-6-sol", }, }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, + cli: { + runtime: "codex", + model: "gpt-6.1-sol", + }, + }, { id: "gpt-6-luna", label: "GPT-6 Luna", @@ -215,6 +245,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [ provider: "deepseek", model: "deepseek-v4-pro", }, + cli: { + runtime: "claude-code", + model: "deepseek-v4-pro", + anthropicEndpoint: { + baseUrl: "https://api.deepseek.com/anthropic", + apiKeyEnv: "DEEPSEEK_API_KEY", + }, + }, }, ]; @@ -253,11 +291,20 @@ export function formatRunModelLabel( model: EvalModelSpec, ): string { if (mode === "cli") { - return `${model.cli!.provider}:${model.cli!.model}`; + return formatCliRunModelLabel(model.cli!); } return `${model.frontend!.provider}:${model.frontend!.model}`; } +// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was +// recorded under; any other pairing names the agent. +export function formatCliRunModelLabel(config: CliEvalModelConfig): string { + if (config.runtime === "claude-code" && !config.anthropicEndpoint) { + return `anthropic:${config.model}`; + } + return `${config.runtime}:${config.model}`; +} + export function getFrontendEvalModel( model: EvalModelSpec, ): FrontendEvalModelConfig { diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index 832a383a73..7e70191992 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -29,6 +29,10 @@ export interface EvalCaseRuntimeAppContextSpec { export interface EvalCaseRuntimeSpec { maxTurns?: number; + // Global mode: scripted answers to askUserQuestion, one per question in order; a question + // past the last one is dismissed, so `[]` dismisses them all. Unset, the question tool + // reports that it cannot ask. + userAnswers?: string[]; backendPreview?: EvalCaseRuntimeBackendPreview; appContext?: EvalCaseRuntimeAppContextSpec; // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. diff --git a/ai_evals/modes/cli.ts b/ai_evals/modes/cli.ts index 205cdfa9c4..f337f99f40 100644 --- a/ai_evals/modes/cli.ts +++ b/ai_evals/modes/cli.ts @@ -39,17 +39,36 @@ interface CliRunActual { trace: CliTrace; } -const CLAUDE_PROJECT_PREAMBLE = [ - "Follow the project instructions from AGENTS.md exactly.", - "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", - "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", - "Do not skip the Skill step.", +// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex +// loads a skill by reading its SKILL.md. +const SKILL_STEP: Record = { + "claude-code": [ + "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the Skill step.", + ], + codex: [ + "Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.", + "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.", + "Do not skip the skill step.", + ], +}; + +const HARNESS_PREAMBLE = [ "You are running inside an automated benchmark harness, not an interactive user session.", "Act autonomously and complete the requested file changes directly in the workspace.", "Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.", "Do not respond with a plan when you can make the change directly.", "Only describe what was done after you have written the files.", -].join(" "); +]; + +function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string { + return [ + "Follow the project instructions from AGENTS.md exactly.", + ...SKILL_STEP[runtime], + ...HARNESS_PREAMBLE, + ].join(" "); +} export function createCliModeRunner( modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL @@ -89,7 +108,7 @@ export function createCliModeRunner( }); await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8"); - const renderedPrompt = await renderPrompt(prompt, workspaceDir); + const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime); const run = await runPromptAndCapture( renderedPrompt, workspaceDir, @@ -193,7 +212,11 @@ export function getCliRunModelLabel( return formatCliRunModelLabel(modelConfig); } -async function renderPrompt(prompt: string, workspaceDir: string): Promise { +async function renderPrompt( + prompt: string, + workspaceDir: string, + runtime: CliEvalModelConfig["runtime"] +): Promise { const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir); const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8"); @@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a" +} diff --git a/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json new file mode 100644 index 0000000000..036610e08c --- /dev/null +++ b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b" +} diff --git a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json similarity index 54% rename from backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json rename to backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json index bb7e4fa36b..e03d995f15 100644 --- a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json +++ b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)", + "query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running", "describe": { "columns": [ { "ordinal": 0, - "name": "exists", + "name": "?column?", "type_info": "Bool" } ], @@ -19,5 +19,5 @@ null ] }, - "hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e" + "hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0" } diff --git a/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json new file mode 100644 index 0000000000..1c8dfcfa57 --- /dev/null +++ b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13" +} diff --git a/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json new file mode 100644 index 0000000000..ba917c82db --- /dev/null +++ b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683" +} diff --git a/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json new file mode 100644 index 0000000000..e1657e156b --- /dev/null +++ b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c" +} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json deleted file mode 100644 index 7f6fe7a81c..0000000000 --- a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" -} diff --git a/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json new file mode 100644 index 0000000000..48963d9ef7 --- /dev/null +++ b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.id FROM v2_job_queue q\n JOIN native_retry_attempt n ON n.job_id = q.id\n JOIN v2_job r ON r.id = q.id\n JOIN v2_job a ON a.id = r.parent_job\n WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429" +} diff --git a/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json new file mode 100644 index 0000000000..9ebf9289c2 --- /dev/null +++ b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET draining = true WHERE worker = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919" +} diff --git a/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json new file mode 100644 index 0000000000..9f51c69616 --- /dev/null +++ b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b" +} diff --git a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json similarity index 80% rename from backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json rename to backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json index 4ebad828df..1670a5009e 100644 --- a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json +++ b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1 AND j.parent_job IS NULL\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", + "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258" + "hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9" } diff --git a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json similarity index 86% rename from backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json rename to backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json index 583d2bcc6b..5ee5c7571d 100644 --- a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json +++ b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -107,6 +107,11 @@ "ordinal": 20, "name": "native_mode", "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "draining", + "type_info": "Bool" } ], "parameters": { @@ -138,8 +143,9 @@ true, true, true, + false, false ] }, - "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca" + "hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5e0f480e08..b5f3d9a306 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6685,9 +6685,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] @@ -9617,7 +9617,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.16+spec-1.1.0", ] [[package]] @@ -13622,9 +13622,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] @@ -13644,21 +13644,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.16+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722" dependencies = [ "indexmap 2.14.2", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333" dependencies = [ "winnow 1.0.4", ] @@ -14829,7 +14829,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -14918,7 +14918,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.826.0" +version = "1.828.0" dependencies = [ "async-stream", "async-trait", @@ -14952,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14965,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "argon2", @@ -15105,7 +15105,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15128,7 +15128,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15145,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15171,7 +15171,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.826.0" +version = "1.828.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15181,7 +15181,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15198,7 +15198,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15259,7 +15259,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15281,7 +15281,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15303,7 +15303,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -15353,7 +15353,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15372,13 +15372,14 @@ dependencies = [ "uuid", "windmill-api-auth", "windmill-api-sse", + "windmill-audit", "windmill-common", "windmill-queue", ] [[package]] name = "windmill-api-npm-proxy" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15406,7 +15407,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15428,7 +15429,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15448,7 +15449,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15490,7 +15491,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15519,7 +15520,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.826.0" +version = "1.828.0" dependencies = [ "lazy_static", "serde", @@ -15531,7 +15532,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.826.0" +version = "1.828.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15555,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15569,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.826.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15608,7 +15609,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.826.0" +version = "1.828.0" dependencies = [ "chrono", "lazy_static", @@ -15617,12 +15618,13 @@ dependencies = [ "sql-builder", "sqlx", "tracing", + "tracing-subscriber", "windmill-common", ] [[package]] name = "windmill-autoscaling" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15641,7 +15643,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.828.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15755,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.826.0" +version = "1.828.0" dependencies = [ "chrono", "futures", @@ -15775,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.826.0" +version = "1.828.0" dependencies = [ "regex", "serde", @@ -15792,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15819,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "futures", @@ -15836,7 +15838,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.828.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15852,7 +15854,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -15873,7 +15875,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -15904,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "arc-swap", @@ -15929,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-stream", @@ -15965,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "futures", @@ -15983,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.828.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15992,7 +15994,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16004,7 +16006,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16016,7 +16018,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "gosyn", @@ -16028,7 +16030,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16040,7 +16042,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16052,7 +16054,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "nu-parser", @@ -16063,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16074,7 +16076,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16086,7 +16088,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16097,7 +16099,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16119,7 +16121,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16131,7 +16133,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16145,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16162,7 +16164,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16175,7 +16177,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde", @@ -16187,7 +16189,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16205,7 +16207,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16221,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16237,7 +16239,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16251,7 +16253,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16290,7 +16292,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "const_format", @@ -16330,7 +16332,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.826.0" +version = "1.828.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16341,7 +16343,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16376,7 +16378,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16400,7 +16402,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16433,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16460,7 +16462,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16493,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16513,7 +16515,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16547,7 +16549,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16584,7 +16586,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16607,7 +16609,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16631,7 +16633,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -16655,7 +16657,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16690,7 +16692,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16718,7 +16720,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16743,7 +16745,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16762,7 +16764,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-once-cell", @@ -16881,7 +16883,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.826.0" +version = "1.828.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7808c0d621..e0b97275b2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.826.0" +version = "1.828.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.826.0" +version = "1.828.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7f8d64aa9d..1acd2de3f0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -065c2c64c3904d5c0b2766b424210d34f9e54d2d \ No newline at end of file +a37413ccdd52e65bc240bad3e87084bf4100e119 diff --git a/backend/migrations/20261008070335_worker_ping_draining.down.sql b/backend/migrations/20261008070335_worker_ping_draining.down.sql new file mode 100644 index 0000000000..64a5f2949d --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.down.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining; diff --git a/backend/migrations/20261008070335_worker_ping_draining.up.sql b/backend/migrations/20261008070335_worker_ping_draining.up.sql new file mode 100644 index 0000000000..33b5ad5e3e --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.up.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 38f4b7a33e..10929e409a 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -1040,11 +1040,13 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ { pub fn parse_snowflake_typ(typ: &str) -> Typ { match typ { - "varchar" => Typ::Str(None), - "binary" => Typ::Bytes, - "date" | "time" | "timestamp" => Typ::Datetime, - "int" => Typ::Int, - "float" => Typ::Float, + "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None), + "binary" | "varbinary" => Typ::Bytes, + "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz" + | "timestamp_tz" => Typ::Datetime, + "int" | "integer" | "bigint" | "smallint" | "tinyint" | "byteint" => Typ::Int, + "float" | "float4" | "float8" | "double" | "double precision" | "real" | "number" + | "decimal" | "numeric" => Typ::Float, "boolean" => Typ::Bool, "s3object" => Typ::Resource("S3Object".to_string()), _ => Typ::Str(None), diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index d7e33313c6..1bdbdd6097 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.828.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.828.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.828.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.828.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ba69f6e380..b2fd850217 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.826.0" +version = "1.828.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 77963b73c6..8d0af4915d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -40,7 +40,7 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts}; use windmill_common::global_settings::OAUTH_SETTING; use windmill_common::otel_oss::{ otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool, - otel_set_queue_count, otel_set_queue_running_count, + otel_set_queue_count, otel_set_queue_delay, otel_set_queue_running_count, }; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, @@ -168,6 +168,12 @@ lazy_static::lazy_static! { &["tag"] ).unwrap(); + static ref QUEUE_DELAY_SECONDS: prometheus::GaugeVec = prometheus::register_gauge_vec!( + "queue_delay_seconds", + "How long the next job to be picked up has been waiting, in seconds", + &["tag"] + ).unwrap(); + static ref QUEUE_RUNNING_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!( "queue_running_count", "Number of running jobs in the queue", @@ -421,6 +427,8 @@ pub async fn initial_load( native_mode, // an agent worker never reads its group's config, only its token object_store_cache_config: None, + // nor whether it is paused: the server refuses it jobs instead + paused: false, })); } } @@ -5597,6 +5605,7 @@ pub async fn expose_queue_metrics(db: &Pool) { for q in QUEUE_COUNT_TAGS.read().await.iter() { if queue_stats.get(q).is_none() { (*QUEUE_COUNT).with_label_values(&[q]).set(0); + (*QUEUE_DELAY_SECONDS).with_label_values(&[q]).set(0.0); } } } @@ -5605,6 +5614,7 @@ pub async fn expose_queue_metrics(db: &Pool) { for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() { if queue_stats.get(q).is_none() { otel_set_queue_count(q, 0); + otel_set_queue_delay(q, 0.0); } } } @@ -5620,6 +5630,9 @@ pub async fn expose_queue_metrics(db: &Pool) { if metrics_enabled { let metric = (*QUEUE_COUNT).with_label_values(&[tag]); metric.set(count as i64); + (*QUEUE_DELAY_SECONDS) + .with_label_values(&[tag]) + .set(stat.delay); tags_to_watch.push(tag.to_string()); } @@ -5627,6 +5640,7 @@ pub async fn expose_queue_metrics(db: &Pool) { otel_tags_to_watch.push(tag.to_string()); } otel_set_queue_count(tag, count as i64); + otel_set_queue_delay(tag, stat.delay); } if save_metrics { @@ -5677,8 +5691,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .as_ref() .is_some_and(|dws| !dws.is_empty()); + // Pausing or resuming must not restart the workers, dedicated ones included: a reload + // whose only difference is the flag skips every restart below. + let only_pause_changed = wc.paused != config.paused + && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc; + if **wc != config || has_dedicated { - if kill_if_change { + if kill_if_change && !only_pause_changed { if has_dedicated || wc.dedicated_worker != config.dedicated_worker || wc.dedicated_workers != config.dedicated_workers diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 2a518a7cc8..284a822b47 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -219,7 +219,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) -worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) +worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool) workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) diff --git a/backend/tests/ai_decision.rs b/backend/tests/ai_decision.rs index 98c219dfb2..04b310f2c1 100644 --- a/backend/tests/ai_decision.rs +++ b/backend/tests/ai_decision.rs @@ -1,5 +1,6 @@ -//! AI decision steps run end to end against a stand-in for System One endpoints: TypeSafe's, and -//! Cloudflare Workers AI's, which serves each model at its own URL inside an API envelope. +//! AI decision steps run end to end against a stand-in for the decision endpoints: TypeSafe's +//! System One, Cloudflare Workers AI's, which serves each model at its own URL inside an API +//! envelope, and OpenAI's Decisions API. use axum::{routing::post, Json, Router}; use serde_json::{json, Value}; @@ -37,6 +38,19 @@ async fn start_decision_stub() -> anyhow::Result { post(|Json(body): Json| async move { Json(json!({"result": refund_answers(&body), "success": true, "errors": []})) }), + ) + // OpenAI's Decisions API takes the questions as a list and answers with one. + .route( + "/openai/v1/decisions", + post(|Json(body): Json| async move { + let answers = body["questions"] + .as_array() + .into_iter() + .flatten() + .map(|q| json!({"type": "choice", "name": q["name"], "choice": "refund"})) + .collect::>(); + Json(json!({"answers": answers})) + }), ); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let port = listener.local_addr()?.port(); @@ -72,6 +86,11 @@ async fn test_ai_decision_answers_route_a_following_branch( }, "model": "clef-flash" }), + json!({ + "kind": "openai", + "resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/openai/v1")}, + "model": "gpt-6-luna" + }), ] { let flow: FlowValue = serde_json::from_value(json!({ "modules": [ diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 4f58685221..b1f740566f 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its failure must reach the workspace error handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_error_handler_triggered_on_hub_script_failure( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000002"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/fail"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(!completed_job.success, "Hub script should have failed"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace error handler should handle the failed hub script" + ); + + Ok(()) +} diff --git a/backend/tests/job_token_scopes.rs b/backend/tests/job_token_scopes.rs index 6af694bff0..f805168bfa 100644 --- a/backend/tests/job_token_scopes.rs +++ b/backend/tests/job_token_scopes.rs @@ -410,5 +410,22 @@ async fn test_oidc_claim_carries_the_presented_scopes(db: Pool) -> any )?; assert_eq!(claims["job_token_scopes"], expected, "{job}"); } + + // A soft-canceled job is still `running` until its worker kills it. + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(Uuid::parse_str(OIDC_JOB)?) + .execute(&db) + .await?; + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, OIDC_JOB).await?) + .send() + .await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); Ok(()) } diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index 1a4040d390..f1e042037f 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -168,6 +168,7 @@ async fn test_all_metrics_e2e() { // ── Gauges ────────────────────────────────────────────────────── otel_set_queue_count("python3", 42); + otel_set_queue_delay("python3", 12.5); otel_set_queue_running_count("deno", 5); otel_set_worker_busy("worker-test-1", 1); otel_set_db_pool(5, 10, 20); @@ -188,7 +189,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 20 metric names are present ────────────────────── + // ── Verify all 21 metric names are present ────────────────────── let expected = [ "windmill.queue.push_count", @@ -197,6 +198,7 @@ async fn test_all_metrics_e2e() { "windmill.queue.zombie_restart_count", "windmill.queue.zombie_delete_count", "windmill.queue.count", + "windmill.queue.delay", "windmill.queue.running_count", "windmill.worker.execution_count", "windmill.worker.execution_duration", @@ -261,6 +263,9 @@ async fn test_all_metrics_e2e() { .expect("queue.count data point with tag=python3 not found"); assert_eq!(dp.1, 42); + let m = find_metric(&metrics, "windmill.queue.delay").unwrap(); + assert!((gauge_f64_value(m).unwrap() - 12.5).abs() < f64::EPSILON); + let m = find_metric(&metrics, "windmill.queue.running_count").unwrap(); let values = gauge_i64_values(m); let dp = values diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index 8fc4182d9e..f876c1984b 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow: Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its success must reach the workspace success handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_hub_script_success( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000003"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/ok"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(completed_job.success, "Hub script should have succeeded"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace success handler should handle the successful hub script" + ); + + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index f420dd2a12..73d3fc2567 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -4698,6 +4698,129 @@ async fn test_run_wait_result_early_return_with_failure_module( Ok(()) } +/// `recover: true` from the error handler of a step that fails inside a loop turns the flow +/// green, as it does for a top-level step, without changing which steps run: a loop that +/// stops at a failed iteration still stops there, one that skips failures still carries on. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let failing_loop = |skip_failures: bool| { + json!({ + "id": "loop", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1, 2, 3] }, + "skip_failures": skip_failures, + "modules": [{ + "id": "a", + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "deno", + "content": "export function main() { throw new Error('boom'); }", + }, + }], + }, + }) + }; + let run = |loop_step: serde_json::Value| { + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [loop_step, { + "id": "b", + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "deno", + "content": "export function main() { return { ran_b: true } }", + }, + }], + "failure_module": { + "value": { + "input_transforms": {}, + "type": "rawscript", + "language": "deno", + "content": "export function main() { return { handled: true, recover: true } }", + }, + }, + })) + .unwrap(); + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + }; + let handler_runs = || { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE j.kind = 'preview' AND c.result @> '{\"handled\": true}'::jsonb", + ) + .fetch_one(&db) + }; + + let stopped = run(failing_loop(false)).await; + let stopped_handler_runs = handler_runs().await?; + let skipping = run(failing_loop(true)).await; + let skipping_handler_runs = handler_runs().await? - stopped_handler_runs; + // A parallel node counts its iterations' stored status: an iteration that ended on a + // recovered failure must still be stored as failed there. + let parallel = run(json!({ + "id": "par", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1, 2] }, + "parallel": true, + "skip_failures": false, + "modules": [failing_loop(false)], + }, + })) + .await; + // A recovery the inner loop absorbs, then an error raised starting the next step: that + // error is a new failure and must leave the flow failed. + #[cfg(feature = "quickjs")] + let chaining_error = run(json!({ + "id": "outer", + "value": { + "type": "forloopflow", + "iterator": { "type": "static", "value": [1] }, + "skip_failures": false, + "modules": [failing_loop(true), { + "id": "guarded", + "skip_if": { "expr": "missingFunction()" }, + "value": { "input_transforms": {}, "type": "identity" }, + }], + }, + })) + .await; + + server.close().await.unwrap(); + + #[cfg(feature = "quickjs")] + assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed"); + + assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success"); + assert!( + stopped.json_result().unwrap().get("ran_b").is_none(), + "the step after the loop must not run, as without recovery" + ); + assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration"); + + assert!(skipping.success); + assert_eq!( + skipping.json_result().unwrap(), + json!({ "ran_b": true }), + "a loop that skips failures runs every iteration and the following step" + ); + assert_eq!(skipping_handler_runs, 3); + + assert!( + !parallel.json_result().unwrap().to_string().contains("ran_b"), + "the parallel loop fails as without recovery, so the step after it must not run" + ); + Ok(()) +} + #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { diff --git a/backend/tests/worker_group_pause.rs b/backend/tests/worker_group_pause.rs new file mode 100644 index 0000000000..b528bbfd77 --- /dev/null +++ b/backend/tests/worker_group_pause.rs @@ -0,0 +1,71 @@ +//! In its own test binary: it flips the process-wide `WORKER_CONFIG`, which would stall every +//! other worker test sharing the process. + +use futures::StreamExt; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, +}; +use windmill_test_utils::*; + +fn set_paused(paused: bool) { + let mut wc = (**WORKER_CONFIG.load()).clone(); + wc.paused = paused; + WORKER_CONFIG.store(std::sync::Arc::new(wc)); +} + +#[sqlx::test(fixtures("base"))] +async fn paused_worker_leaves_jobs_queued_until_resumed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + set_paused(true); + let id = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "echo done".to_string(), + path: None, + lock: None, + language: ScriptLang::Bash, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker( + Connection::Sql(db.clone()), + async { + tokio::time::sleep(Duration::from_secs(3)).await; + let running = + sqlx::query_scalar::<_, bool>("SELECT running FROM v2_job_queue WHERE id = $1") + .bind(id) + .fetch_optional(&db) + .await + .unwrap(); + assert_eq!( + running, + Some(false), + "a paused worker must not pull the job" + ); + + set_paused(false); + while completed.next().await != Some(id) {} + }, + port, + ) + .await; + + assert!(completed_job(id, &db).await.success); + Ok(()) +} diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs index 772f4193c3..2421e888ef 100644 --- a/backend/tests/worker_ping_ip.rs +++ b/backend/tests/worker_ping_ip.rs @@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres}; use uuid::Uuid; use windmill_common::{ external_ip::UNKNOWN_IP, - worker::{insert_ping_query, update_worker_ping_main_loop_query}, + worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query}, }; async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + register(db, worker, ip, None).await +} + +async fn register( + db: &Pool, + worker: &str, + ip: Option<&str>, + draining: Option, +) -> anyhow::Result<()> { insert_ping_query( "test-instance", worker, @@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> any None, None, false, + draining, db, ) .await?; Ok(()) } +/// A worker re-registers mid-life once its external IP resolves. That registration must not +/// clear a draining flag the same process already published; only a worker starting up under +/// a reclaimed name resets it. +#[sqlx::test] +async fn only_a_starting_worker_resets_draining(db: Pool) -> anyhow::Result<()> { + let draining = || async { + sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'") + .fetch_one(&db) + .await + }; + register(&db, "wk-drain", None, Some(false)).await?; + set_worker_draining_query("wk-drain", &db).await?; + register(&db, "wk-drain", Some("1.2.3.4"), None).await?; + assert!(draining().await?); + + register(&db, "wk-drain", None, Some(false)).await?; + assert!(!draining().await?); + Ok(()) +} + /// The external IP resolves in the background, so the initial ping often has none yet. That must /// not blank the address a previous process wrote to the row this one reclaims — worker names are /// stable across restarts under EXIT_AFTER_N_JOBS. diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index 4cd41f34a8..3b3af833d6 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -124,7 +124,7 @@ pub enum AIProvider { #[serde(rename = "aws_bedrock")] AWSBedrock, /// TypeSafe's Jev decision model. It answers typed questions rather than chatting, so only - /// an AI decision runs it (`run_systemone`), never the agent loop. + /// an AI decision runs it (`run_decision`), never the agent loop. TypeSafe, /// Cloudflare Workers AI, for its Jev-compatible decision models (Clef). Decisions only, as /// for TypeSafe; its base URL comes from the resource's account id. @@ -152,6 +152,12 @@ impl AIProvider { matches!(self, AIProvider::TypeSafe | AIProvider::Cloudflare) } + /// A provider an AI decision runs on: the decision providers, and OpenAI, whose Decisions + /// API sits next to its chat routes on the same resource. + pub fn runs_decisions(&self) -> bool { + self.is_decision_provider() || matches!(self, AIProvider::OpenAI) + } + /// Get the base URL for the AI provider pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { if let Some(base_url) = resource_base_url { diff --git a/backend/windmill-ai/src/decision.rs b/backend/windmill-ai/src/decision.rs index 31b525cf2d..b3a35dab2a 100644 --- a/backend/windmill-ai/src/decision.rs +++ b/backend/windmill-ai/src/decision.rs @@ -1,6 +1,7 @@ -//! AI decision steps: one call to a System One endpoint (TypeSafe's Jev, or Cloudflare's -//! Jev-compatible Clef on Workers AI), which answers typed questions (`choice`, `score`, `noul`) -//! about a state with calibrated probabilities instead of text. +//! AI decision steps: one call to a decision endpoint, which answers typed questions (`choice`, +//! `score`, `noul`) about a state with calibrated probabilities instead of text. The questions and +//! answers are System One's (TypeSafe's Jev, Cloudflare's Jev-compatible Clef on Workers AI); +//! OpenAI's Decisions API is asked and read through a translation to that shape. use std::time::Duration; @@ -30,6 +31,8 @@ pub struct AIDecisionArgs { pub const TYPESAFE_DEFAULT_MODEL: &str = "jev-latest"; /// Cloudflare's larger decision model, run when the step names none. pub const CLOUDFLARE_DEFAULT_MODEL: &str = "clef"; +/// The model OpenAI's Decisions API serves, run when the step names none. +pub const OPENAI_DEFAULT_MODEL: &str = "gpt-6-luna"; #[derive(Serialize)] struct SystemOneRequest<'a> { @@ -108,7 +111,263 @@ pub fn decision_inputs<'a>( Ok((state, questions)) } -pub async fn run_systemone( +/// Posts a decision request with the resource's bearer key and returns the response body. +async fn post_decision( + credentials: &ProviderCredentials, + endpoint: &str, + vendor: &str, + payload: &T, + timeout: Duration, +) -> Result { + let auth_headers = retain_effective_credentials( + credentials, + vec![( + "Authorization", + format!( + "Bearer {}", + credentials.api_key.as_deref().unwrap_or_default() + ), + )], + ); + + // The endpoint derives from the resource's base URL, so the connect is pinned to the + // address the SSRF check validated. + let client = pinned_ai_client_for(&credentials.base_url).await?; + let mut request = client.post(endpoint).timeout(timeout); + for (name, value) in auth_headers { + request = request.header(name, value); + } + for (name, value) in common_outbound_headers(credentials) { + request = request.header(name, value); + } + + let response = request + .json(payload) + .send() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?; + + let status = response.status(); + if !status.is_success() { + let body = response + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::ExecutionErr(format!( + "{vendor} error calling {endpoint}: {status} - {body}" + ))); + } + + response + .text() + .await + .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}"))) +} + +/// Answers a decision on the endpoint its provider serves. +pub async fn run_decision( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + if credentials.provider == AIProvider::OpenAI { + run_openai_decisions(credentials, model, state, questions, timeout).await + } else { + run_systemone(credentials, model, state, questions, timeout).await + } +} + +#[derive(Serialize)] +struct OpenAIDecisionsRequest<'a> { + model: &'a str, + input: Value, + questions: Vec, +} + +#[derive(Deserialize)] +struct OpenAIDecisionsResponse { + #[serde(default)] + model: Option, + answers: Vec>, + #[serde(default)] + usage: Option, +} + +/// The state as a Decisions `input`. OpenAI reads a text or a list of user messages, so a list of +/// user messages (the only way to pass an image) goes as it is and any other object or array, +/// a chat history included, as its JSON text. +fn openai_input(state: &Value) -> Value { + match state { + Value::String(_) => state.clone(), + Value::Array(items) if !items.is_empty() && items.iter().all(is_openai_user_message) => { + state.clone() + } + _ => Value::String(state.to_string()), + } +} + +/// Exactly a message the Decisions API reads: `role: "user"` with a text, or with parts that are +/// each an `input_text` or an `input_image`. +fn is_openai_user_message(message: &Value) -> bool { + let is_part = |part: &Value| match part.get("type").and_then(Value::as_str) { + Some("input_text") => part.get("text").is_some_and(Value::is_string), + Some("input_image") => part.get("image_url").is_some_and(Value::is_string), + _ => false, + }; + message.get("role").and_then(Value::as_str) == Some("user") + && match message.get("content") { + Some(Value::String(_)) => true, + Some(Value::Array(parts)) => !parts.is_empty() && parts.iter().all(is_part), + _ => false, + } +} + +/// The questions as OpenAI's Decisions API takes them: a list naming each question, with a +/// `noul` asked as its `predicate` and each score level as a label. +fn openai_questions(questions: &Map) -> Result> { + let text = |v: &Value| match v { + Value::String(s) => s.clone(), + v => v.to_string(), + }; + // A structured description goes as its JSON text, as structured instructions do. + let stated = |description: Option<&Value>| match description { + None | Some(Value::Null) => None, + Some(Value::String(d)) if d.trim().is_empty() => None, + Some(d) => Some(text(d)), + }; + questions + .iter() + .map(|(name, question)| { + let criteria = question.get("criteria"); + let mut instructions = question.get("instructions").map(text).unwrap_or_default(); + let mut asked = Map::new(); + let kind = match question.get("type").and_then(Value::as_str) { + Some("choice") => { + let choices = criteria + .and_then(Value::as_object) + .filter(|c| !c.is_empty()) + .ok_or_else(|| { + Error::BadRequest(format!( + "The choice question '{name}' needs its options as 'criteria'" + )) + })? + .iter() + .map(|(value, d)| match stated(Some(d)) { + Some(d) => serde_json::json!({"value": value, "description": d}), + None => serde_json::json!({"value": value}), + }) + .collect(); + asked.insert("choices".to_string(), Value::Array(choices)); + "choice" + } + Some("score") => { + let levels = criteria + .and_then(Value::as_array) + .filter(|c| !c.is_empty()) + .ok_or_else(|| { + Error::BadRequest(format!( + "The score question '{name}' needs its levels as 'criteria'" + )) + })? + .iter() + .map(|level| serde_json::json!({"label": text(level)})) + .collect(); + asked.insert("levels".to_string(), Value::Array(levels)); + "score" + } + Some("noul") => { + for (key, meaning) in [("true", "Yes"), ("false", "No")] { + if let Some(d) = stated(criteria.and_then(|c| c.get(key))) { + instructions.push_str(&format!("\n{meaning}: {d}")); + } + } + "predicate" + } + other => { + return Err(Error::BadRequest(format!( + "The question '{name}' has type {}, which is not 'choice', 'score' or 'noul'", + other.map_or("none".to_string(), |t| format!("'{t}'")) + ))) + } + }; + asked.insert("type".to_string(), kind.into()); + asked.insert("name".to_string(), name.as_str().into()); + asked.insert("instructions".to_string(), instructions.into()); + Ok(Value::Object(asked)) + }) + .collect() +} + +/// OpenAI's answers keyed by question name, as every decision's are. A predicate's probability is +/// also given as `noul`, so a flow reads a yes/no answer the same way whichever provider gave it. +/// A refused question fails the decision: left in the output it has no `choice` or `noul`, so a +/// branch on it would quietly take its "no" or default arm. +fn openai_answers(answers: Vec>) -> Result> { + let mut by_name = Map::new(); + for mut answer in answers { + let Some(Value::String(name)) = answer.remove("name") else { + continue; + }; + match answer.get("type").and_then(Value::as_str) { + Some("refusal") => { + return Err(Error::ExecutionErr(format!( + "OpenAI refused to answer the question '{name}': {}", + Value::Object(answer) + ))) + } + Some("predicate") => { + if let Some(probability) = answer.get("probability").cloned() { + answer.insert("noul".to_string(), probability); + } + } + _ => {} + } + by_name.insert(name, Value::Object(answer)); + } + Ok(by_name) +} + +async fn run_openai_decisions( + credentials: &ProviderCredentials, + model: &str, + state: &Value, + questions: &Map, + timeout: Duration, +) -> Result { + let model = match model.trim() { + "" => OPENAI_DEFAULT_MODEL, + model => model, + }; + let endpoint = format!("{}/decisions", credentials.base_url.trim_end_matches('/')); + let body = post_decision( + credentials, + &endpoint, + "OpenAI", + &OpenAIDecisionsRequest { + model, + input: openai_input(state), + questions: openai_questions(questions)?, + }, + timeout, + ) + .await?; + let parsed = serde_json::from_str::(&body) + .map_err(|e| Error::ExecutionErr(format!("Unexpected OpenAI response ({e}): {body}")))?; + + Ok(DecisionResult { + output: serde_json::value::to_raw_value(&openai_answers(parsed.answers)?) + .map_err(|e| Error::internal_err(format!("serializing decision answers: {e}")))?, + model: parsed.model, + usage: parsed + .usage + .map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens)) + .filter(|u| !u.is_empty()), + }) +} + +async fn run_systemone( credentials: &ProviderCredentials, model: &str, state: &Value, @@ -138,49 +397,14 @@ pub async fn run_systemone( format!("{base}/systemone") }; let vendor = if cloudflare { "Cloudflare" } else { "TypeSafe" }; - let auth_headers = retain_effective_credentials( + let body = post_decision( credentials, - vec![( - "Authorization", - format!( - "Bearer {}", - credentials.api_key.as_deref().unwrap_or_default() - ), - )], - ); - - // The endpoint derives from the resource's base URL, so the connect is pinned to the - // address the SSRF check validated. - let client = pinned_ai_client_for(&credentials.base_url).await?; - let mut request = client.post(&endpoint).timeout(timeout); - for (name, value) in auth_headers { - request = request.header(name, value); - } - for (name, value) in common_outbound_headers(credentials) { - request = request.header(name, value); - } - - let response = request - .json(&SystemOneRequest { model, state, questions }) - .send() - .await - .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?; - - let status = response.status(); - if !status.is_success() { - let body = response - .text() - .await - .unwrap_or_else(|_| "".to_string()); - return Err(Error::ExecutionErr(format!( - "{vendor} error calling {endpoint}: {status} - {body}" - ))); - } - - let body = response - .text() - .await - .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}")))?; + &endpoint, + vendor, + &SystemOneRequest { model, state, questions }, + timeout, + ) + .await?; // Tried in turn rather than as an untagged enum, which cannot hold the raw `answers`. let parsed = match serde_json::from_str::(&body) { Ok(parsed) => parsed, @@ -223,6 +447,80 @@ mod tests { } } + /// A `noul` goes out as a predicate with its yes/no meanings in the instructions, and comes + /// back readable as `noul`; choices and levels go out as lists, answers come back by name. + #[test] + fn openai_questions_and_answers_translate() { + let questions = json!({ + "intent": {"type": "choice", "instructions": "Why?", "criteria": {"refund": {"when": "paid"}, "bug": ""}}, + "urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["Can wait", "Now"]}, + "angry": {"type": "noul", "instructions": "Angry?", "criteria": {"true": "Insults"}} + }); + let asked = Value::Array(openai_questions(questions.as_object().unwrap()).unwrap()); + assert_eq!( + asked, + json!([ + {"choices": [ + {"value": "refund", "description": "{\"when\":\"paid\"}"}, + {"value": "bug"} + ], "type": "choice", "name": "intent", "instructions": "Why?"}, + {"levels": [ + {"label": "Can wait"}, + {"label": "Now"} + ], "type": "score", "name": "urgency", "instructions": "How urgent?"}, + {"type": "predicate", "name": "angry", "instructions": "Angry?\nYes: Insults"} + ]) + ); + for bad in [ + json!({"q": {"type": "choice", "instructions": "?"}}), + json!({"q": {"type": "score", "instructions": "?", "criteria": []}}), + json!({"q": {"type": "predicate", "instructions": "?"}}), + ] { + assert!(openai_questions(bad.as_object().unwrap()).is_err(), "{bad}"); + } + + let answers = serde_json::from_value(json!([ + {"type": "predicate", "name": "angry", "probability": 0.92}, + {"type": "choice", "name": "intent", "choice": "refund", "confidence": 0.9} + ])) + .unwrap(); + assert_eq!( + Value::Object(openai_answers(answers).unwrap()), + json!({ + "angry": {"type": "predicate", "probability": 0.92, "noul": 0.92}, + "intent": {"type": "choice", "choice": "refund", "confidence": 0.9} + }) + ); + + let messages = json!([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]); + assert_eq!(openai_input(&messages), messages); + assert_eq!(openai_input(&json!({"m": "hi"})), json!("{\"m\":\"hi\"}")); + assert_eq!( + openai_input(&json!([{"role": "admin"}])), + json!("[{\"role\":\"admin\"}]") + ); + for records in [ + json!([{"role": "user", "content": {"ticket": "charged twice"}}]), + json!([{"role": "user", "content": null}]), + json!([{"role": "user", "content": ["a"]}]), + json!([{"role": "user", "content": [{"type": "input_text", "text": {"t": 1}}]}]), + json!([{"role": "user", "content": [{"type": "input_image"}]}]), + ] { + assert_eq!(openai_input(&records), json!(records.to_string())); + } + assert_eq!( + openai_input(&json!([{"role": "user", "content": "Hi"}])), + json!([{"role": "user", "content": "Hi"}]) + ); + // A chat history is evaluated as text: OpenAI takes user messages only. + let history = + json!([{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Yo"}]); + assert_eq!(openai_input(&history), json!(history.to_string())); + let refused = + serde_json::from_value(json!([{"type": "refusal", "name": "angry"}])).unwrap(); + assert!(openai_answers(refused).is_err()); + } + /// Refused are the values a flow produces when it forgot the input (no key, `null`, a blank /// string from an empty expression) and the scalars TypeSafe rejects. A text, an object or an /// array is a state, whatever it holds. diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index e08b038bb0..7dcddc93a3 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -62,9 +62,13 @@ const REASONING_RULES: &[ReasoningRule] = &[ can_disable: false, completions_tools_need_off: false, }, - // Live-verified: astra takes low..max only, where sol and luna also take `none`. + // Live-verified: gpt-6-astra and gpt-6.1-sol take low..max only, where gpt-6-sol and + // gpt-6-luna also take `none`. ReasoningRule { - matches: |m| base_id(m).starts_with("gpt-6-astra"), + matches: |m| { + let base = base_id(m); + base.starts_with("gpt-6-astra") || base.starts_with("gpt-6.1-sol") + }, can_disable: false, completions_tools_need_off: true, }, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index dce6a0ff72..b636e11a1c 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -996,6 +996,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui ["job_metrics", "set_progress", ..] => (http_method == "POST", 2), ["job_metrics", "get_progress", ..] => (http_method == "GET", 2), ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2), + ["jobs", "worker_is_draining", ..] => (http_method == "GET", 2), ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3), ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3), _ => return false, diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7598021f00..f1f4792e71 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -224,12 +224,13 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode + // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), "init_bash": config.get("init_bash"), - "native_mode": config.get("native_mode") + "native_mode": config.get("native_mode"), + "paused": config.get("paused") }) } else { config @@ -272,6 +273,18 @@ async fn update_config( } let mut tx = db.begin().await?; + let was_paused = if name.starts_with("worker__") { + sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .unwrap_or(false) + } else { + false + }; sqlx::query!( "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", &name, @@ -291,6 +304,17 @@ async fn update_config( ) .await?; tx.commit().await?; + + if name.starts_with("worker__") { + let paused = config.get("paused").and_then(|v| v.as_bool()) == Some(true); + if paused != was_paused { + windmill_common::feature_usage::log_feature_usage( + "worker_group", + "pause_toggle", + if paused { "pause" } else { "resume" }, + ); + } + } Ok(format!("Updated config {name}")) } diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index ddb94f8c2b..9800d97841 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -417,6 +417,89 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_existing_role_is_taken_over_only_when_asked_and_never_with_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let plain = format!("wmtest_plain_{suffix}"); + let group = format!("wmtest_group_{suffix}"); + let member = format!("wmtest_member_{suffix}"); + for sql in [ + // An expired password and no connection slot: kept as is, the login Windmill sets is dead. + format!( + "CREATE ROLE \"{plain}\" LOGIN PASSWORD 'theirs' VALID UNTIL '2000-01-01' \ + CONNECTION LIMIT 0" + ), + format!("CREATE ROLE \"{group}\" NOLOGIN"), + format!("CREATE ROLE \"{member}\" LOGIN"), + format!("GRANT \"{group}\" TO \"{member}\""), + ] { + sqlx::query(&sql).execute(&db).await?; + } + + let create = |name: String, take_over: bool| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name, "take_over": take_over })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status().as_u16(), resp.text().await?)) + }; + let outcome = async { + let (status, body) = create(plain.clone(), false).await?; + assert_eq!( + status, 409, + "an existing role is not taken without asking: {body}" + ); + + // The group's member would inherit every grant the data table role gets. + let (status, body) = create(group.clone(), true).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains(&member), + "the refusal names the member: {body}" + ); + + let (status, body) = create(plain.clone(), true).await?; + assert_eq!(status, 200, "{body}"); + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; + let taken = catalog + .values() + .find(|r| r.name == plain) + .ok_or_else(|| anyhow::anyhow!("{plain} is not in the catalog: {catalog:?}"))?; + let options = (*db.connect_options()) + .clone() + .username(&plain) + .password(taken.pwd.as_deref().unwrap_or_default()); + let mut login = ::connect_with(&options).await?; + sqlx::query("SELECT 1").execute(&mut login).await?; + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide and outlive this test's database, whatever happened above. + for name in [&member, &group, &plain] { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + #[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs index c3da2f3fb9..2c5d2ee78e 100644 --- a/backend/windmill-api-integration-tests/tests/jobs_authed.rs +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue( Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_worker_is_draining_follows_the_job_worker(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker) + VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version) + VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')", + ) + .execute(&db) + .await?; + + let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}"); + let draining = || async { + let resp = authed(client().get(&url)).send().await?; + anyhow::Ok(resp.json::().await?) + }; + assert!(!draining().await?); + + windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?; + assert!(draining().await?); + + // A parked job keeps its last worker on the queue row without running on it. + sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1") + .bind(job) + .execute(&db) + .await?; + assert!(!draining().await?); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/Cargo.toml b/backend/windmill-api-jobs/Cargo.toml index d3e10205ac..dad6469947 100644 --- a/backend/windmill-api-jobs/Cargo.toml +++ b/backend/windmill-api-jobs/Cargo.toml @@ -15,6 +15,7 @@ enterprise = ["windmill-common/enterprise"] [dependencies] windmill-api-auth.workspace = true windmill-api-sse.workspace = true +windmill-audit.workspace = true windmill-common = { workspace = true, default-features = false } windmill-queue.workspace = true axum.workspace = true diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index c35b9e41b0..ce6cfc185d 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -23,12 +23,17 @@ use std::sync::Arc; use tokio::sync::RwLock; use uuid::Uuid; use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed}; +use windmill_audit::{ + audit_oss::{audit_log_many, AuditAuthorable}, + ActionKind, +}; use windmill_common::{ db::{UserDB, UserDbWithAuthed}, error::{self, Error}, flow_conversations::{ add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType, }, + flows::Retry, get_latest_flow_version_info_for_path, jobs::{ check_tag_available_for_workspace_internal, format_result, script_path_to_payload, @@ -122,6 +127,71 @@ pub async fn drop_unclaimable_run_lineage( Ok(()) } +/// Applies the run's `retry` policy by handing `push` a one-step-flow request, which it +/// materializes as a native retryable `Script`. +pub fn with_run_retry( + run_query: &RunJobQuery, + payload: JobPayload, + args: &PushArgs<'_>, + tag: &Option, +) -> error::Result { + let Some(retry) = run_query.retry.as_deref() else { + return Ok(payload); + }; + // Retry attempts are always pushed visible to the owner, which would leak a private run. + if run_query.invisible_to_owner.unwrap_or(false) { + return Err(Error::BadRequest( + "retry cannot be combined with invisible_to_owner".to_string(), + )); + } + let retry: Retry = serde_json::from_str(retry) + .map_err(|e| Error::BadRequest(format!("invalid retry policy: {e}")))?; + let JobPayload::ScriptHash { + hash, + path, + cache_ttl, + cache_ignore_s3_path, + language, + priority, + apply_preprocessor, + debouncing_settings, + concurrency_settings, + .. + } = payload + else { + return Err(Error::BadRequest( + "retry is only supported for workspace scripts".to_string(), + )); + }; + // A failed attempt keeps its raw args and the retry re-push skips the preprocessor, so + // every retry would call `main` with un-preprocessed input. + if apply_preprocessor { + return Err(Error::BadRequest( + "retry is not supported for a script with a preprocessor unless skip_preprocessor is set" + .to_string(), + )); + } + Ok(JobPayload::SingleStepFlow { + path, + hash: Some(hash), + flow_version: None, + language: Some(language), + args: HashMap::from(args), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl, + cache_ignore_s3_path, + priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor, + concurrency_settings, + debouncing_settings, + }) +} + /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for /// a workspace admin anything outside the workspace. A restricted job token never gets the admin @@ -196,11 +266,15 @@ pub async fn check_license_key_valid() -> error::Result<()> { pub async fn cancel_jobs( jobs: Vec, db: &DB, - username: &str, + author: &(impl AuditAuthorable + Sync), w_id: &str, force_cancel: bool, ) -> error::JsonResult> { + let username = author.username(); let mut uuids = vec![]; + // A job that completed before its turn is reported back like the others but was not + // cancelled, so it gets no audit entry. + let mut audited = vec![]; tracing::info!("Cancelling jobs: {:?}", jobs); let mut tx = db.begin().await?; let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj @@ -246,7 +320,7 @@ pub async fn cancel_jobs( } match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { let tx = db.begin().await?; - let (tx, _) = cancel_job( + let (tx, cancelled) = cancel_job( username, None, job_id.clone(), @@ -258,13 +332,14 @@ pub async fn cancel_jobs( ) .await?; tx.commit().await?; - Ok::<_, anyhow::Error>(()) + Ok::<_, anyhow::Error>(cancelled) }) .await { Ok(result) => match result { - Ok(_) => { + Ok(cancelled) => { uuids.push(job_id); + audited.extend(cancelled); } Err(e) => { tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); @@ -279,8 +354,27 @@ pub async fn cancel_jobs( } } + audited.extend(trivial_jobs.iter().copied()); uuids.extend(trivial_jobs); + // The cancels are committed by now: failing here would report them as not done. + if let Err(e) = audit_log_many( + db, + author, + if force_cancel { + "jobs.force_cancel" + } else { + "jobs.cancel" + }, + ActionKind::Delete, + w_id, + &audited.iter().map(|id| id.to_string()).collect::>(), + ) + .await + { + tracing::error!("Failed to write audit entries for cancelled jobs in {w_id}: {e:#}"); + } + Ok(Json(uuids)) } @@ -1123,6 +1217,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( let tag = run_query.tag.clone().or(tag); let push_args = PushArgs { args: &args.args, extra: args.extra }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; let return_tx = tx_o.is_some(); diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index f37040bd11..f50cb61451 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -52,6 +52,8 @@ pub struct RunJobQuery { pub trigger_external_id: Option, pub service_name: Option, pub suspended_mode: Option, + /// A JSON-encoded `Retry` policy for a script run; see `with_run_retry`. + pub retry: Option, } impl RunJobQuery { diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 7a523c89fa..855f687431 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1487,11 +1487,12 @@ async fn set_default_error_handler( } }; - if let Some(value_content) = value { - windmill_api_settings::set_global_setting_internal(&db, key, value_content).await?; + if let Some(value_content) = value.clone() { + windmill_api_settings::set_global_setting_internal(&db, key.clone(), value_content).await?; } else { windmill_api_settings::delete_global_setting(&db, key.as_str()).await?; } + windmill_api_settings::audit_setting_write(&db, &authed, &key, value.as_ref()).await?; if payload.override_existing { // The rewrite and its history rows go in one transaction: on separate diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 7e2ec536c0..c79b596617 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -29,6 +29,7 @@ mod log_cleanup; mod storage_usage; use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; use windmill_common::DB; @@ -829,7 +830,71 @@ pub async fn set_global_setting( Json(value): Json, ) -> error::Result<()> { require_super_admin(&db, &authed).await?; - set_global_setting_internal(&db, key, value.value.unwrap_or(serde_json::Value::Null)).await + let value = value.value.unwrap_or(serde_json::Value::Null); + set_global_setting_internal(&db, key.clone(), value.clone()).await?; + audit_setting_write(&db, &authed, &key, Some(&value)).await +} + +/// A setting is recorded by name, with its value only when that is a boolean or a number. +/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no +/// list of secret-bearing settings stays complete as settings are added. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `authed` may change instance settings before performing the write this records. +pub async fn audit_setting_write( + db: &DB, + authed: &ApiAuthed, + key: &str, + value: Option<&serde_json::Value>, +) -> error::Result<()> { + let deleted = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::String(s)) => s.trim().is_empty(), + Some(_) => false, + }; + let recorded = match value { + Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { + Some(v.to_string()) + } + _ => None, + }; + audit_log( + db, + authed, + if deleted { + "global_settings.delete" + } else { + "global_settings.update" + }, + if deleted { + ActionKind::Delete + } else { + ActionKind::Update + }, + "global", + Some(key), + recorded.as_deref().map(|v| [("value", v)].into()), + ) + .await +} + +async fn audit_instance_action( + db: &DB, + authed: &ApiAuthed, + operation: &str, + action_kind: ActionKind, + parameters: Option>, +) -> error::Result<()> { + audit_log( + db, + authed, + operation, + action_kind, + "global", + Some(&authed.email), + parameters, + ) + .await } pub async fn set_global_setting_internal( @@ -1305,15 +1370,61 @@ async fn set_instance_config( run_setting_pre_write_hook(&db, key, value).await?; } } + // The external-instance write removes its own key from the diff. + let external = settings_diff + .upserts + .get(EXTERNAL_INSTANCE_PG_SETTING) + .cloned() + .map(Some) + .or_else(|| { + settings_diff + .deletes + .iter() + .any(|k| k == EXTERNAL_INSTANCE_PG_SETTING) + .then_some(None) + }); windmill_common::external_instance_pg::write_external_instance_pg_from_diff( &db, &mut settings_diff, ) .await?; + if let Some(value) = external { + audit_setting_write(&db, &authed, EXTERNAL_INSTANCE_PG_SETTING, value.as_ref()).await?; + } - instance_config::apply_settings_diff(&db, &settings_diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; + // Applied and audited one setting at a time: the writes are not transactional, so a + // batch that fails midway must not leave the settings it did write unrecorded. + let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } = + settings_diff; + let steps = upserts + .into_iter() + .map(|(k, v)| (k, Some(v))) + .chain(deletes.into_iter().map(|k| (k, None))) + .map(Some) + .chain(std::iter::once(None)); + for step in steps { + let mut diff = instance_config::SettingsDiff::default(); + match &step { + Some((key, value)) => { + if let Some(previous) = previous_values.remove(key) { + diff.previous_values.insert(key.clone(), previous); + } + match value { + Some(value) => { + diff.upserts.insert(key.clone(), value.clone()); + } + None => diff.deletes.push(key.clone()), + } + } + None => diff.unchanged_count = unchanged_count, + } + instance_config::apply_settings_diff(&db, &diff) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; + if let Some((key, value)) = &step { + audit_setting_write(&db, &authed, key, value.as_ref()).await?; + } + } if ai_config_changed { bump_instance_ai_config_revision(); @@ -1343,9 +1454,37 @@ async fn set_instance_config( .collect(); let configs_diff = instance_config::diff_worker_configs(¤t_wc, &desired_wc, ApplyMode::Merge); - instance_config::apply_configs_diff(&db, &configs_diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; + let steps = configs_diff + .upserts + .into_iter() + .map(|(g, c)| (g, Some(c))) + .chain(configs_diff.deletes.into_iter().map(|g| (g, None))); + for (group, config) in steps { + let mut diff = instance_config::ConfigsDiff::default(); + let (operation, action_kind) = match config { + Some(config) => { + diff.upserts.insert(group.clone(), config); + ("worker_config.update", ActionKind::Update) + } + None => { + diff.deletes.push(group.clone()); + ("worker_config.delete", ActionKind::Delete) + } + }; + instance_config::apply_configs_diff(&db, &diff) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; + audit_log( + &db, + &authed, + operation, + action_kind, + "global", + Some(&format!("worker__{group}")), + None, + ) + .await?; + } } Ok(()) @@ -1583,6 +1722,14 @@ pub async fn renew_license_key( } ))); } else { + audit_instance_action( + &db, + &authed, + "settings.renew_license_key", + ActionKind::Update, + None, + ) + .await?; return Ok("Renewed license key".to_string()); } } @@ -1812,6 +1959,14 @@ async fn refresh_custom_instance_user_pwd( require_super_admin(&db, &authed).await?; windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?; windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?; + audit_instance_action( + &db, + &authed, + "settings.refresh_custom_instance_user_pwd", + ActionKind::Update, + None, + ) + .await?; Ok(Json(())) } @@ -1970,7 +2125,8 @@ async fn setup_custom_instance_pg_database( windmill_common::datatable_roles::lock_instance_databases_governance(&mut tx, [dbname.trim()]) .await?; let mut logs = CustomInstanceDbLogs::default(); - let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; + let result = + setup_custom_instance_pg_database_inner(authed.clone(), &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); let status = CustomInstanceDb { @@ -1998,6 +2154,20 @@ async fn setup_custom_instance_pg_database( .fetch_one(&mut *tx) .await?; tx.commit().await?; + audit_instance_action( + &db, + &authed, + "settings.setup_custom_instance_pg_database", + ActionKind::Create, + Some( + [ + ("dbname", dbname.as_str()), + ("success", if success { "true" } else { "false" }), + ] + .into(), + ), + ) + .await?; let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?; Ok(Json(status)) @@ -2176,6 +2346,14 @@ async fn drop_custom_instance_pg_database( require_super_admin(&db, &authed).await?; windmill_common::drop_custom_instance_database(&db, &dbname).await?; + audit_instance_action( + &db, + &authed, + "settings.drop_custom_instance_pg_database", + ActionKind::Delete, + Some([("dbname", dbname.as_str())].into()), + ) + .await?; Ok(format!("Database '{}' dropped successfully", dbname)) } @@ -2220,6 +2398,14 @@ pub async fn migrate_secrets_to_vault( let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_vault", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2241,6 +2427,14 @@ pub async fn migrate_secrets_to_database( let report = windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_database", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2274,6 +2468,14 @@ pub async fn migrate_secrets_to_azure_kv( let report = windmill_common::secret_backend::migrate_secrets_to_azure_kv(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_azure_kv", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2291,6 +2493,14 @@ pub async fn migrate_secrets_from_azure_kv( let report = windmill_common::secret_backend::migrate_secrets_from_azure_kv(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_from_azure_kv", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2315,6 +2525,14 @@ pub async fn migrate_secrets_to_aws_sm( ) -> JsonResult { require_super_admin(&db, &authed).await?; let report = windmill_common::secret_backend::migrate_secrets_to_aws_sm(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_to_aws_sm", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } @@ -2328,6 +2546,14 @@ pub async fn migrate_secrets_from_aws_sm( require_super_admin(&db, &authed).await?; let report = windmill_common::secret_backend::migrate_secrets_from_aws_sm(&db, &settings).await?; + audit_instance_action( + &db, + &authed, + "settings.migrate_secrets_from_aws_sm", + ActionKind::Update, + None, + ) + .await?; Ok(Json(report)) } diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 0102f1b033..cf0b85609a 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -86,6 +86,7 @@ struct WorkerPing { job_isolation: Option, #[serde(skip_serializing_if = "Option::is_none")] native_mode: Option, + draining: bool, } // #[derive(Serialize, Deserialize)] @@ -118,7 +119,7 @@ async fn list_worker_pings( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index ebab6214ce..efaa712481 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -683,6 +683,9 @@ pub(crate) struct CatalogFacts { pub(crate) former_owner: Option, /// For a revoke: each grant it takes back. pub(crate) revoked_grants: Vec, + /// For a grant on what a schema holds: the role cannot reach the schema yet, so the grant + /// would give it nothing it can use. + pub(crate) lacks_schema_usage: bool, } /// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the @@ -1526,7 +1529,24 @@ async fn build_plan( ))); } } - let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let lacks_schema_usage = match (&change, target) { + (AclChange::Grant { scope, .. }, AclTarget::Schema { schema }) + if !matches!(scope, GrantScope::Target) => + { + !has_schema_usage(client, schema, &pg_role).await? + } + (AclChange::Grant { .. }, AclTarget::Table { schema, .. }) => { + !has_schema_usage(client, schema, &pg_role).await? + } + _ => false, + }; + let facts = CatalogFacts { + other_pg_roles, + existing_objects, + former_owner, + revoked_grants, + lacks_schema_usage, + }; let mut plan = crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; if matches!(change, AclChange::SetOwner { .. }) { @@ -1540,6 +1560,27 @@ async fn build_plan( Ok(plan) } +/// Whether `pg_role` may look into `schema`, directly or through a role it inherits from. +async fn has_schema_usage( + client: &tokio_postgres::Client, + schema: &str, + pg_role: &str, +) -> Result { + Ok(client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'USAGE')", + &[&pg_role, &schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the role's privileges on schema {schema}: {}", + pg_error_message(&e) + )) + })? + .get(0)) +} + /// Postgres only hands an object to a role that could have created it: a table to one with /// `CREATE` on its schema, a schema to one with `CREATE` on the database. async fn missing_owner_privilege( diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index c94560314c..efc108453d 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -10513,7 +10513,7 @@ async fn edit_workspace( pub(crate) async fn archive_workspace_impl( db: &DB, w_id: &str, - username: &str, + authed: &ApiAuthed, // When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the // prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the // archive — a later failure can't strand a half-archived dev that's still flagged/locked. @@ -10627,7 +10627,7 @@ pub(crate) async fn archive_workspace_impl( let axum::Json(canceled_jobs) = windmill_api_jobs::cancel_jobs( jobs_to_cancel, db, - username, + authed, w_id, false, // force_cancel ) @@ -10687,7 +10687,7 @@ async fn archive_workspace( // dev workspace, which only applies to a workspace that is itself a dev (`dev_lock_parent`): a // root archived out from under its dev is the pre-existing shape and not this pairing's to police. let (schedules_count, canceled_count, deleted_tokens_count) = - archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?; + archive_workspace_impl(&db, &w_id, &authed, dev_lock_parent.as_deref()).await?; // Audit log let mut tx = db.begin().await?; diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index a0ff1f2370..50833bfe06 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -944,7 +944,7 @@ pub(crate) async fn change_workspace_id( // Note: schedules were already moved to new workspace, so this will find 0 schedules info!("Archiving old workspace"); let (_schedules_count, canceled_count, _deleted_tokens_count) = - archive_workspace_impl(&db, &old_id, &authed.username, None).await?; + archive_workspace_impl(&db, &old_id, &authed, None).await?; // The old id stays live between the commit above and the archive, and a fork copy created for // it in that window registers under it. Creation checks the workspace is live under the fork diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 8dae1bafcd..4016222d30 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -34993,7 +34993,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -35006,7 +35006,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -47930,7 +47930,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -47943,7 +47943,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 439e4d11a0..d541144231 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -11725,9 +11725,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: >- - Retry with exponential backoff (delay - doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -11738,7 +11736,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -23272,7 +23270,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -23283,7 +23281,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e0d526f8d5..5e635e39bd 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.826.0 + version: 1.828.0 title: Windmill API contact: @@ -1765,6 +1765,12 @@ paths: type: string cluster: $ref: "#/components/schemas/DatatableRoleCluster" + take_over: + type: boolean + description: > + manage a Postgres role of that name that already exists on the cluster, + replacing its password, instead of refusing it. A role with cluster-wide + attributes or memberships is refused either way responses: "200": description: the created role @@ -1772,6 +1778,8 @@ paths: application/json: schema: $ref: "#/components/schemas/InstanceDatatableRole" + "409": + description: a Postgres role of that name already exists on the cluster and take_over was not set /settings/datatable_roles/{id}: post: @@ -12004,6 +12012,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -16008,6 +16017,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) @@ -17312,6 +17322,27 @@ paths: schema: type: string + /w/{workspace}/jobs/worker_is_draining/{id}: + get: + summary: whether the worker running this job is draining + description: | + True once the worker running the job has received its shutdown signal. The worker + still waits for the job to finish, so a long-running job can poll this to exit early. + False when the job is not currently running on a worker. + operationId: workerIsDraining + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: whether the worker running this job is draining + content: + application/json: + schema: + type: boolean + /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs @@ -27822,6 +27853,12 @@ components: in: query schema: type: string + RunRetry: + name: retry + description: JSON-encoded retry policy (same shape as a schedule's retry) re-running the job when it fails. The returned id is the first attempt's; each retry is a job of its own, recorded as a child of the first attempt. Not supported for hub scripts, with invisible_to_owner, or for a script whose preprocessor would run. + in: query + schema: + type: string NewJobId: name: job_id description: @@ -34525,6 +34562,11 @@ components: type: boolean cluster: $ref: "#/components/schemas/DatatableRoleCluster" + in_use: + type: boolean + description: >- + Whether any data table, in any workspace, lets someone connect as this role. Only set + in a data table's available_roles. DatatableRoleTenants: type: object @@ -35880,6 +35922,9 @@ components: type: string native_mode: type: boolean + draining: + type: boolean + description: the worker received its shutdown signal and exits once its current job is done required: - worker - worker_instance diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f276e6d143..26b369ce74 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -979,7 +979,14 @@ async fn get_raw_app_data( // the opaque, sandboxed wrapper (incl. under a cross-origin-isolated // / COEP `require-corp` embedder). .header("X-Content-Type-Options", "nosniff") - .header("Cross-Origin-Resource-Policy", "cross-origin"); + .header("Cross-Origin-Resource-Policy", "cross-origin") + // The secret names one app version, whose bundle is written once at deploy, + // so the URL's content never changes. `private`: the URL is a capability, so + // shared caches must not keep serving it after the app is deleted. + .header( + http::header::CACHE_CONTROL, + "private, max-age=31536000, immutable", + ); Ok(res.body(body).unwrap()) } else { return Err(Error::NotFound("File not found".to_string())); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 1f92c6d425..5b0da21c50 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -378,6 +378,10 @@ pub fn workspaced_service() -> Router { "/wac_approval_urls/{job_id}/{step_key}", get(get_wac_approval_urls).layer(cors.clone()), ) + .route( + "/worker_is_draining/{id}", + get(worker_is_draining).layer(cors.clone()), + ) .route( "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), @@ -613,6 +617,53 @@ async fn get_root_job( Ok(Json(res)) } +/// Whether the worker running this job has received its shutdown signal. A draining worker +/// never interrupts its job, so a long-running script polls this to exit on its own terms. +async fn worker_is_draining( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> windmill_common::error::JsonResult { + // A job always may ask about itself. Anyone else must be able to see the job: the + // caller's row policies on v2_job decide, narrowed by the token's tag filter. + let draining = if authed.job_id == Some(id) { + sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker + WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + id, + w_id + ) + .fetch_optional(&db) + .await? + } else { + let tags = get_scope_tags(&authed) + .map(|tags| tags.into_iter().map(str::to_string).collect::>()); + let mut tx = user_db.begin(&authed).await?; + let draining = sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job j + JOIN v2_job_queue q ON q.id = j.id + JOIN worker_ping wp ON wp.worker = q.worker + WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL + AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_deref() + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + draining + } + .unwrap_or(false); + windmill_common::feature_usage::log_feature_usage( + "worker_draining", + "check", + if draining { "draining" } else { "not_draining" }, + ); + Ok(Json(draining)) +} + async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, @@ -4580,7 +4631,7 @@ async fn cancel_selection( let Json(mut w_cancelled) = cancel_jobs( ids, &db, - authed.username.as_str(), + &authed, workspace_id.as_str(), force_cancel, ) @@ -8541,6 +8592,11 @@ pub async fn stream_job( run_query: RunJobQuery, is_get: bool, ) -> error::Result { + if run_query.retry.is_some() { + return Err(error::Error::BadRequest( + "retry is only supported when running a script asynchronously".to_string(), + )); + } let args = if is_get { let payload_as_args = run_query.payload_as_args()?; @@ -10483,26 +10539,29 @@ pub async fn run_job_by_hash_inner( ) }; + let job_payload = JobPayload::ScriptHash { + hash: ScriptHash(hash), + path: path, + concurrency_settings, + debouncing_settings, + cache_ttl, + cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + labels, + job_token_scopes, + }; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, &w_id, - JobPayload::ScriptHash { - hash: ScriptHash(hash), - path: path, - concurrency_settings, - debouncing_settings, - cache_ttl, - cache_ignore_s3_path, - language, - dedicated_worker, - priority, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - labels, - job_token_scopes, - }, + job_payload, push_args, authed.display_username(), email, diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 1962710578..73b1fd1792 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -2024,7 +2024,7 @@ You should get the schema of the script or flow before creating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2037,7 +2037,7 @@ You should get the schema of the script or flow before creating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -2231,7 +2231,7 @@ You should get the schema of the script or flow before updating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2244,7 +2244,7 @@ You should get the schema of the script or flow before updating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-audit/Cargo.toml b/backend/windmill-audit/Cargo.toml index 6e4fc3fd12..8c26615664 100644 --- a/backend/windmill-audit/Cargo.toml +++ b/backend/windmill-audit/Cargo.toml @@ -20,4 +20,7 @@ chrono.workspace = true serde_json.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } -lazy_static.workspace = true \ No newline at end of file +lazy_static.workspace = true + +[dev-dependencies] +tracing-subscriber.workspace = true \ No newline at end of file diff --git a/backend/windmill-audit/src/audit_oss.rs b/backend/windmill-audit/src/audit_oss.rs index da36ada468..02b075af9d 100644 --- a/backend/windmill-audit/src/audit_oss.rs +++ b/backend/windmill-audit/src/audit_oss.rs @@ -88,6 +88,21 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>( Ok(()) } +/// AUTHORIZATION: records entries, checks nothing. The caller must have authorized `author` +/// for the operation on every resource in `w_id` before calling. +#[cfg(not(feature = "private"))] +pub async fn audit_log_many<'c, E: sqlx::Executor<'c, Database = Postgres>>( + _db: E, + _author: &impl AuditAuthorable, + _operation: &str, + _action_kind: ActionKind, + _w_id: &str, + _resources: &[String], +) -> Result<()> { + // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature + Ok(()) +} + #[cfg(not(feature = "private"))] pub async fn list_audit( _tx: Transaction<'_, Postgres>, diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index f355e733cb..e7e490a6a4 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -331,6 +331,12 @@ pub async fn converge_connect_grants_with( /// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user` on `cluster`. No /// privileges beyond that — an admin grants them through SQL or the ACL editor. /// +/// A role of that name that already exists is refused with a 409, unless `take_over`: then its +/// password is replaced and it is managed from here on like one created here. A role holding +/// cluster-wide privileges, or inheriting another role's, is refused either way, since every tenant +/// would get them. Returns whether an existing role was taken over: undoing a failed take-over +/// must not drop a role that was there before. +/// /// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external /// cluster is another server: the role is created there before `tx` commits, and callers MUST drop /// it again ([`drop_datatable_role`]) if `tx` then fails to commit. @@ -343,8 +349,10 @@ pub async fn create_datatable_role( cluster: DatatableRoleCluster, name: &str, password: &str, -) -> Result<()> { - crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await + take_over: bool, +) -> Result { + crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password, take_over) + .await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index a97a936827..33d711e7f9 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -125,7 +125,8 @@ mod ce { _cluster: DatatableRoleCluster, _name: &str, _password: &str, - ) -> Result<()> { + _take_over: bool, + ) -> Result { Err(unavailable()) } diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index a4164cd2c5..700ba71df5 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -963,6 +963,9 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Workers of the group pull no new job while this is set; queued jobs wait. + #[serde(skip_serializing_if = "Option::is_none")] + pub paused: Option, /// Object store this group's dependency cache uses instead of the instance one. Same shape /// as the instance `object_store_cache_config`. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-common/src/log_context.rs b/backend/windmill-common/src/log_context.rs index cdb1e27aea..c5cd191da8 100644 --- a/backend/windmill-common/src/log_context.rs +++ b/backend/windmill-common/src/log_context.rs @@ -49,6 +49,7 @@ pub struct LogContext { // Worker / job span (windmill-worker/src/worker.rs) pub worker: Option, + pub worker_group: Option, pub hostname: Option, pub tag: Option, pub job_id: Option, diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index 5bdab37528..bd0a5202df 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -94,6 +94,9 @@ pub fn otel_incr_zombie_delete_count(_count: u64) {} #[cfg(not(feature = "private"))] pub fn otel_set_queue_count(_tag: &str, _count: i64) {} +#[cfg(not(feature = "private"))] +pub fn otel_set_queue_delay(_tag: &str, _delay_secs: f64) {} + #[cfg(not(feature = "private"))] pub fn otel_set_queue_running_count(_tag: &str, _count: i64) {} diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index b456e370fc..01e75ac489 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -44,6 +44,11 @@ lazy_static::lazy_static! { /// Use `tracing::info!(target: windmill_common::tracing_init::VERBOSE_TARGET, ...)` for logs that should be suppressed in quiet mode. pub const VERBOSE_TARGET: &str = "windmill_verbose"; +/// Target of audit events, so a log pipeline can route them without matching on content. +/// It keeps the `windmill` prefix because that is what a `RUST_LOG` of `info` or `debug` +/// is rewritten to filter on. +pub const AUDIT_TARGET: &str = "windmill:audit"; + /// Prefix used by user scripts to emit a log line as an OTEL tracing event /// when `OTEL_JOB_LOGS=true`. Stripped before forwarding to the tracing layer. pub const OTEL_PREFIX: &str = "OTEL: "; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index e68b81e4de..52fa2da905 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -477,6 +477,7 @@ lazy_static::lazy_static! { env_vars: Default::default(), native_mode: false, object_store_cache_config: Default::default(), + paused: false, }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -844,11 +845,14 @@ pub fn make_suspended_pull_query(tags: &[String]) -> String { } // pub async fn make_suspended pub async fn store_suspended_pull_query(wc: &WorkerConfig) { - if wc.worker_tags.len() == 0 { - tracing::error!("Empty tags in worker tags, skipping"); - return; - } - let query = make_suspended_pull_query(&wc.worker_tags); + // An empty tag list must clear the query: keeping the previous one would leave a reloaded + // worker pulling suspended jobs for tags it no longer listens to. + let query = if wc.worker_tags.is_empty() { + tracing::error!("Empty tags in worker tags, pulling no suspended jobs"); + String::new() + } else { + make_suspended_pull_query(&wc.worker_tags) + }; WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query)); } @@ -922,6 +926,42 @@ pub async fn store_pull_query(wc: &WorkerConfig) { WORKER_PULL_QUERIES_FAIRNESS.store(std::sync::Arc::new(fairness_queries)); } +const WORKER_GROUP_PAUSED_CACHE_TTL: Duration = Duration::from_secs(10); + +lazy_static::lazy_static! { + static ref WORKER_GROUP_PAUSED_CACHE: quick_cache::sync::Cache = + quick_cache::sync::Cache::new(1000); +} + +/// Whether the config of `worker_group` is paused, as the server sees it. A worker that reads +/// its own config knows from `WORKER_CONFIG`; this is for the server deciding on behalf of +/// workers that do not, and is cached since they ask on every pull. +pub async fn is_worker_group_paused(db: &DB, worker_group: &str) -> bool { + if let Some((paused, at)) = WORKER_GROUP_PAUSED_CACHE.get(worker_group) { + if at.elapsed() < WORKER_GROUP_PAUSED_CACHE_TTL { + return paused; + } + } + let paused = match sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + format!("worker__{worker_group}") + ) + .fetch_optional(db) + .await + { + Ok(paused) => paused.flatten().unwrap_or(false), + Err(e) => { + tracing::error!("Could not read whether worker group {worker_group} is paused: {e:#}"); + return false; + } + }; + WORKER_GROUP_PAUSED_CACHE.insert( + worker_group.to_string(), + (paused, std::time::Instant::now()), + ); + paused +} + lazy_static::lazy_static! { pub static ref WINDMILL_DIR: String = { let dir = std::env::var("WINDMILL_DIR") @@ -1912,6 +1952,7 @@ pub enum PingType { MainLoop, Job, InitScript, + Draining, } #[derive(Serialize, Deserialize)] pub struct Ping { @@ -1934,6 +1975,8 @@ pub struct Ping { pub occupancy_rate_30m: Option, pub job_isolation: Option, pub native_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draining: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -1986,6 +2029,7 @@ pub async fn update_ping_http( insert_ping.memory, insert_ping.job_isolation, insert_ping.native_mode.unwrap_or(false), + insert_ping.draining, db, ) .await?; @@ -2014,10 +2058,21 @@ pub async fn update_ping_http( ) .await? } + PingType::Draining => set_worker_draining_query(worker_name, db).await?, } Ok(()) } +pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET draining = true WHERE worker = $1", + worker_name + ) + .execute(db) + .await?; + Ok(()) +} + #[derive(Debug, Serialize, Deserialize)] pub struct JobCancelled { pub canceled_by: String, @@ -2116,6 +2171,9 @@ pub async fn fetch_raw_script_from_app_query( /// `started_at` and `jobs_executed` are the only two columns carried over, being the /// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP /// lookup has not resolved yet and the predecessor's address is still the best guess. +/// +/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering +/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime. pub async fn insert_ping_query( worker_instance: &str, worker_name: &str, @@ -2129,6 +2187,7 @@ pub async fn insert_ping_query( memory: Option, job_isolation: Option, native_mode: bool, + draining: Option, db: &DB, ) -> anyhow::Result { // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and @@ -2136,8 +2195,8 @@ pub async fn insert_ping_query( // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The // literal below must stay equal to `external_ip::UNKNOWN_IP`. let previous_jobs_executed = sqlx::query_scalar!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) - DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker) + DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining) RETURNING jobs_executed", worker_instance, worker_name, @@ -2151,6 +2210,7 @@ pub async fn insert_ping_query( memory, job_isolation.as_deref(), native_mode, + draining, ) .fetch_one(db) .await?; @@ -2470,6 +2530,12 @@ pub async fn load_worker_config( .collect(); priority_tags_sorted.push(PriorityTags { priority: 0, tags: Vec::from_iter(all_tags_set) }); // push the tags that were not listed as high priority with a priority = 0 priority_tags_sorted.sort_by_key(|elt| Reverse(elt.priority)); // sort by priority DESC + + // The buckets come out of hash maps: without a fixed order two loads of the same + // config compare unequal, and a reload reads that as a config change. + for bucket in priority_tags_sorted.iter_mut() { + bucket.tags.sort(); + } } else { // if no priority is used, push all tags with a priority to 0 priority_tags_sorted.push(PriorityTags { priority: 0, tags: worker_tags.clone() }); @@ -2539,6 +2605,7 @@ pub async fn load_worker_config( env_vars: resolved_env_vars, native_mode, object_store_cache_config: config.object_store_cache_config, + paused: config.paused.unwrap_or(false), }) } @@ -2629,6 +2696,7 @@ pub struct WorkerConfigOpt { pub env_vars_allowlist: Option>, pub native_mode: Option, pub object_store_cache_config: Option, + pub paused: Option, } impl Default for WorkerConfigOpt { @@ -2648,6 +2716,7 @@ impl Default for WorkerConfigOpt { env_vars_allowlist: Default::default(), native_mode: Default::default(), object_store_cache_config: Default::default(), + paused: Default::default(), } } } @@ -2670,12 +2739,15 @@ pub struct WorkerConfig { /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate /// that parses it, and comparing the raw value is what tells a reload the store changed. pub object_store_cache_config: Option, + /// The group pulls no job from the queue: its workers finish what they hold, keep pinging + /// and leave queued jobs for when it is resumed. + pub paused: bool, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, self.paused, // holds bucket credentials self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 207c8ee176..e8e3716d2b 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -359,6 +359,22 @@ ORDER BY depth, id .filter_map(|r| r.id.clone()) .collect_vec(); + // A native retry is the child of its first attempt, which has completed by the time the + // retry is queued, so the walk above stops before reaching it. + let tree = [&[job.id][..], &jobs_to_cancel[..]].concat(); + let pending_retries = sqlx::query_scalar!( + "SELECT q.id FROM v2_job_queue q + JOIN native_retry_attempt n ON n.job_id = q.id + JOIN v2_job r ON r.id = q.id + JOIN v2_job a ON a.id = r.parent_job + WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + tree.as_slice(), + w_id + ) + .fetch_all(&mut *tx) + .await?; + jobs_to_cancel.extend(pending_retries); + jobs_to_cancel.reverse(); if !jobs_to_cancel.is_empty() { tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len()); @@ -1391,7 +1407,18 @@ pub async fn add_completed_job( .then(|| completed_job.parent_job) .flatten() } else if !success && !skipped && retry_pending { - Some(completed_job.parent_job.unwrap_or(completed_job.id)) + let is_retry_attempt = sqlx::query_scalar!( + "SELECT attempt FROM native_retry_attempt WHERE job_id = $1", + completed_job.id, + ) + .fetch_optional(db) + .await + .unwrap_or_else(|e| { + tracing::error!("reading the retry marker of {}: {e:#}", completed_job.id); + None + }) + .is_some(); + Some(native_retry_root(completed_job, is_retry_attempt)) } else { None }; @@ -2194,18 +2221,17 @@ async fn restart_job_if_perpetual_inner( /// resolution too; `ON CONFLICT DO NOTHING` keeps a human's note intact. /// /// Membership of the chain must be *proven* per row, never inferred from `parent_job` -/// alone: `root` is `job.parent_job.unwrap_or(job.id)`, so for a job launched with an -/// explicit `parent_job` (WAC inline children, SDK-launched children) `root` is the -/// *calling* job, and its other failed children are unrelated. Resolving those would hide -/// exactly the failures this feature exists to surface, so each row must be either +/// alone: the success-side caller passes the succeeding job's `parent_job`, which for a +/// job launched with an explicit parent (WAC inline children, SDK-launched children) is +/// the *calling* job, and its other failed children are unrelated. Resolving those would +/// hide exactly the failures this feature exists to surface, so each row must be either /// - a job carrying a `native_retry_attempt` marker under `root` (provably an attempt), or -/// - `root` itself as the original attempt, which is only provable when `root` is -/// parentless and unmarked. +/// - `root` itself as the original attempt: unmarked, and proven a chain root by the +/// marked success under it, since only a chain root is ever a marked job's parent +/// (see [`native_retry_root`]). /// /// Both arms also require the same `runnable_id` as the succeeding attempt, since every -/// attempt in a chain runs the same runnable. The deliberate cost is a miss, not an -/// over-reach: when the original attempt had a parent it is an unmarked sibling -/// indistinguishable from any other child of the caller, so it stays red. +/// attempt in a chain runs the same runnable. /// /// Nothing is trusted of the caller (this writes through a privileged pool): the gate is /// "some marked attempt under `root` running `runnable_id` has succeeded", evaluated in @@ -2249,7 +2275,7 @@ pub async fn resolve_retry_chain_if_succeeded( AND ( (j.parent_job = $1 AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) - OR (c.id = $1 AND j.parent_job IS NULL + OR (c.id = $1 AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) ) ON CONFLICT (job_id) DO NOTHING", @@ -2303,6 +2329,18 @@ async fn eval_retry_if( false } +/// The first attempt of `job`'s native retry chain, which every later attempt is pushed +/// with as `parent_job` and whose id seeds theirs. A first attempt may have a parent of its +/// own (a script dispatched from another job), so only a marked attempt's parent is the root: +/// rooting at that outer parent would merge the chains of its retried children. +fn native_retry_root(job: &MiniCompletedJob, is_retry_attempt: bool) -> Uuid { + if is_retry_attempt { + job.parent_job.unwrap_or(job.id) + } else { + job.id + } +} + /// Native script retry. When a failed `Script` or `Script_Hub` job carries a retry /// policy (via `runnable_settings_handle`) and has attempts left, enqueue a fresh /// attempt of the same script after the policy's backoff delay — instead of having @@ -2346,6 +2384,18 @@ pub async fn maybe_enqueue_native_script_retry( if !policy.has_attempts() { return Ok(false); } + // Cancelling a job that has a parent (every retry attempt does) only flags its queue row, + // and the worker that pulls it completes it without a `canceled_by`: read the flag here. + let soft_canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + job.id, + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if soft_canceled { + return Ok(false); + } // Attempt counter for this job: its `native_retry_attempt` marker, or 0 for // the first (un-marked) attempt. The marker is persistent (unlike the queue @@ -2358,7 +2408,7 @@ pub async fn maybe_enqueue_native_script_retry( .fetch_optional(db) .await? .unwrap_or(0) as u32; - let root = job.parent_job.unwrap_or(job.id); + let root = native_retry_root(job, prev_attempts > 0); // Scheduled chains keep the schedule trigger so the terminal attempt drives // the schedule completion handlers (on_failure/on_success); `parent_job` // keeps every retry out of the per-occurrence handler counting queries. diff --git a/backend/windmill-queue/tests/concurrency_counter_test.rs b/backend/windmill-queue/tests/concurrency_counter_test.rs index 3edb0d3b72..b341c317bb 100644 --- a/backend/windmill-queue/tests/concurrency_counter_test.rs +++ b/backend/windmill-queue/tests/concurrency_counter_test.rs @@ -33,32 +33,19 @@ mod concurrency { .unwrap_or(0) } - /// Stress: N concurrent admit-attempts on a pre-existing key with limit - /// L < N. Asserts admitted count is exactly L (never over-admits) AND - /// that the row's job_uuids reflects exactly L entries. + /// Stress: N concurrent admit-attempts on a key with limit L < N. Asserts + /// admitted count is exactly L (never over-admits) AND that the row's + /// job_uuids reflects exactly L entries. /// - /// The row is pre-seeded with an empty `job_uuids` because there is a - /// known preexisting race when the `concurrency_counter` row does not - /// exist yet: `SELECT ... FOR UPDATE` on a missing row locks nothing, - /// so all concurrent transactions see `running=0` and all admit via - /// `INSERT ... ON CONFLICT DO UPDATE`. That race is inherited from - /// the legacy code path and is out of scope for this PR. This test - /// validates the property we *do* preserve: no over-admit once the - /// row exists. + /// The `concurrency_counter` row is deliberately absent at the start: the + /// monitor deletes empty rows while jobs for the key are still queued, so + /// the admit path has to serialize without a pre-existing row to lock. #[sqlx::test(migrations = "../migrations")] async fn stress_never_over_admits(db: Pool) { let key = "stress-never-over-admits".to_string(); let limit: i32 = 5; let workers: usize = 50; - sqlx::query!( - "INSERT INTO concurrency_counter(concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb)", - key, - ) - .execute(&db) - .await - .expect("seed empty counter row"); - let mut tasks = Vec::with_capacity(workers); for _ in 0..workers { let db = db.clone(); diff --git a/backend/windmill-queue/tests/native_retry_test.rs b/backend/windmill-queue/tests/native_retry_test.rs index 9b0e05a74c..4cf278cf83 100644 --- a/backend/windmill-queue/tests/native_retry_test.rs +++ b/backend/windmill-queue/tests/native_retry_test.rs @@ -561,4 +561,123 @@ mod native_retry { .unwrap(); assert_eq!(status, "failure", "resolving must not change job status"); } + + // A script dispatched from another job keeps that job as its parent, so its chain is + // rooted at the first attempt itself: sibling dispatches must not share retry ids, and + // a parented first attempt is still resolved once its retry succeeds. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn parented_first_attempts_root_their_own_chains( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 1, seconds: 0 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let dispatcher = Uuid::new_v4(); + let (a, b) = (Uuid::new_v4(), Uuid::new_v4()); + for first in [a, b] { + let attempt = mini(first, Some(dispatcher), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + } + assert!( + retry_by_attempt(&db, a, 1).await.is_some(), + "a's retry hangs off a" + ); + assert!( + retry_by_attempt(&db, b, 1).await.is_some(), + "b's retry is not mistaken for a's" + ); + + let (c, c_retry, c_sibling) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + seed_job(&db, c, Some(dispatcher), false, "failure").await; + seed_job(&db, c_sibling, Some(dispatcher), false, "failure").await; + seed_job(&db, c_retry, Some(c), true, "success").await; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, c, WS, None).await?; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, dispatcher, WS, None).await?; + let resolved: Vec = sqlx::query_scalar("SELECT job_id FROM job_resolution") + .fetch_all(&db) + .await?; + assert_eq!( + resolved, + vec![c], + "only the parented first attempt resolves, not its sibling" + ); + Ok(()) + } + + // Cancelling the job that dispatched a script reaches that script's pending retry, which + // hangs off a first attempt that has already completed. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn cancelling_the_caller_cancels_a_pending_retry( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 2, seconds: 60 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let (caller, first) = (Uuid::new_v4(), Uuid::new_v4()); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, kind, runnable_path) VALUES ($1, $2, 'script', $3)", + ) + .bind(caller) + .bind(WS) + .bind(SCRIPT) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running) VALUES ($1, $2, now(), true)", + ) + .bind(caller) + .bind(WS) + .execute(&db) + .await?; + seed_job(&db, first, Some(caller), false, "failure").await; + let attempt = mini(first, Some(caller), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + let (retry_id, ..) = retry_by_attempt(&db, first, 1) + .await + .expect("retry is queued"); + + let tx = db.begin().await?; + let (tx, _) = + windmill_queue::jobs::cancel_job("test-user", None, caller, WS, tx, &db, false, false) + .await?; + tx.commit().await?; + + let canceled_by: Option = + sqlx::query_scalar("SELECT canceled_by FROM v2_job_queue WHERE id = $1") + .bind(retry_id) + .fetch_one(&db) + .await?; + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + // The worker that pulls the canceled retry completes it without a `canceled_by`. + let (_, _, _, _, retry_handle) = retry_by_attempt(&db, first, 1).await.unwrap(); + let canceled_retry = mini(retry_id, Some(first), retry_handle); + assert!( + !maybe_enqueue_native_script_retry(&db, &canceled_retry, &None, &no_result).await?, + "a canceled retry does not enqueue the next attempt" + ); + Ok(()) + } } diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 1c2a76cda5..75aacbcc64 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -929,8 +929,8 @@ pub async fn eval_fetch_timeout( for line in logged.as_deref().unwrap_or(&log).lines() { tracing::info!( target: "windmill:job_log", - job_id = ?job_id, - workspace_id = ?w_id_for_tracing, + job_id = %job_id, + workspace_id = %w_id_for_tracing, "{line}" ); if *OTEL_JOB_LOGS { diff --git a/backend/windmill-trigger/src/global_handler.rs b/backend/windmill-trigger/src/global_handler.rs index ec74b4f139..64ad693a10 100644 --- a/backend/windmill-trigger/src/global_handler.rs +++ b/backend/windmill-trigger/src/global_handler.rs @@ -342,7 +342,7 @@ pub async fn cancel_suspended_trigger_jobs( let cancelled_jobs = cancel_jobs( jobs_to_cancel, &db, - authed.username.as_str(), + &authed, w_id.as_str(), true, ) diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 3f71191415..890318b8aa 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -1140,7 +1140,7 @@ pub enum FlowModuleValue { #[serde(default, skip_serializing_if = "HashMap::is_empty")] tool_inputs: HashMap>, }, - /// One call to a decision model (TypeSafe's Jev) answering the typed `questions` of its + /// One call to a decision model (such as TypeSafe's Jev) answering the typed `questions` of its /// `input_transforms` about a `state`. A flow branches on the answers with a `BranchOne`. AIDecision { input_transforms: HashMap, diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index e2e4ab866b..364795510a 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -54,11 +54,45 @@ pub struct VolumeMount { pub target: String, } +/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while +/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A +/// write within the filesystem's timestamp granularity of the hand-off is not seen. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileStamp { + size: u64, + changed: Option<(i64, i64)>, +} + +impl FileStamp { + /// Whether the file is as it was at `earlier`. An unreadable change time never matches. + pub fn unchanged_since(&self, earlier: &FileStamp) -> bool { + self.changed.is_some() && self == earlier + } + + pub fn of(meta: &std::fs::Metadata) -> Self { + // ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it. + #[cfg(unix)] + let changed = { + use std::os::unix::fs::MetadataExt; + Some((meta.ctime(), meta.ctime_nsec())) + }; + #[cfg(not(unix))] + let changed = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64)); + Self { size: meta.len(), changed } + } +} + pub struct VolumeState { pub mount: VolumeMount, pub local_dir: PathBuf, pub manifest: HashMap, pub symlinks: HashMap, + /// Stamp of each pulled file as the job received it, keyed like `manifest`. + pub stamps: HashMap, } pub struct DownloadStats { diff --git a/backend/windmill-worker/nsjail/build.csharp.config.proto b/backend/windmill-worker/nsjail/build.csharp.config.proto new file mode 100644 index 0000000000..a926d923f2 --- /dev/null +++ b/backend/windmill-worker/nsjail/build.csharp.config.proto @@ -0,0 +1,128 @@ +name: "csharp restore/build" + +mode: ONCE +hostname: "csharp" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "{DOTNET_ROOT}" + dst: "{DOTNET_ROOT}" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# project files, nuget.config, and the published output dir are all here; cwd of dotnet +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# DOTNET_CLI_HOME + NUGET_PACKAGES (nuget/). Writable, like the install/run jails' +# shared caches. +mount { + src: "{CSHARP_CACHE_DIR}" + dst: "{CSHARP_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.php.config.proto b/backend/windmill-worker/nsjail/lock.php.config.proto new file mode 100644 index 0000000000..d06c1699a6 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.php.config.proto @@ -0,0 +1,124 @@ +name: "php composer install" + +mode: ONCE +hostname: "php" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# composer.json/composer.lock live here and vendor/ is written here; cwd of composer +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# COMPOSER_HOME: package download cache. Writable, like the install/run jails' +# shared caches. +mount { + src: "{COMPOSER_CACHE_DIR}" + dst: "{COMPOSER_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +{CA_MOUNT} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.py.config.proto b/backend/windmill-worker/nsjail/lock.py.config.proto new file mode 100644 index 0000000000..14119c4089 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.py.config.proto @@ -0,0 +1,133 @@ +name: "python lock compile" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: {TIMEOUT} + +# Dependency resolution may invoke the PEP 517 build backend of an sdist whose +# metadata is not static, which runs arbitrary package code (e.g. setup.py). +# That build, and uv's bytecode compile pool, size thread/fd pools to the host's +# CPU count, so these mirror download.py.config.proto rather than the run sandbox. +rlimit_as: 8192 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 10000 + +envar: "HOME=/tmp" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# requirements.in -> requirements.txt, cwd of uv +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# shared uv cache: resolution metadata + where sdist build backends run +# (builds-v0/). Writable, as the install jail's target already is; a poisoned +# shared cache is the known residual, not a new one. +mount { + src: "{UV_CACHE_DIR}" + dst: "{UV_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" + is_bind: true +} + +{INDEX_CERT_MOUNT} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +#{DEV} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 133a0a0b0d..7dd2382731 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -28,7 +28,7 @@ use windmill_mcp::McpClient; use crate::ai::tools::McpClientStub as McpClient; use windmill_ai::{ ai_providers::AIProvider, - decision::{decision_inputs, run_systemone, AIDecisionArgs}, + decision::{decision_inputs, run_decision, AIDecisionArgs}, image_handler::upload_image_to_s3, model_context::model_context_window, providers::{ @@ -2449,9 +2449,9 @@ async fn handle_ai_decision( worker_name: &str, ) -> error::Result> { let args = serde_json::from_str::(&serde_json::to_string(local_args)?)?; - if !args.provider.kind.is_decision_provider() { + if !args.provider.kind.runs_decisions() { return Err(Error::BadRequest(format!( - "An AI decision runs on a TypeSafe or Cloudflare resource, not {:?}", + "An AI decision runs on a TypeSafe, Cloudflare or OpenAI resource, not {:?}", args.provider.kind ))); } @@ -2467,7 +2467,7 @@ async fn handle_ai_decision( conn, mem_peak, canceled_by, - run_systemone( + run_decision( &credentials, args.provider.get_model(), state, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 01523eba59..745f202826 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script( wm_memory_usage: None, job_isolation: None, native_mode: None, + draining: None, ping_type: PingType::InitScript, }, ) diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index d393171f7f..7d77f8703f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -33,7 +33,7 @@ use crate::{ get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, CSHARP_CACHE_DIR, - DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, + DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; #[cfg(feature = "csharp")] @@ -47,6 +47,9 @@ use crate::SYSTEM_ROOT; #[cfg(feature = "csharp")] const NSJAIL_CONFIG_RUN_CSHARP_CONTENT: &str = include_str!("../nsjail/run.csharp.config.proto"); +#[cfg(feature = "csharp")] +const NSJAIL_CONFIG_BUILD_CSHARP_CONTENT: &str = + include_str!("../nsjail/build.csharp.config.proto"); #[cfg(feature = "csharp")] #[cfg(windows)] @@ -124,6 +127,9 @@ pub async fn generate_nuget_lockfile( gen_cs_proj(code, job_dir, reqs, lines_to_remove)?; + // `dotnet restore` resolves and downloads packages but does not import their + // MSBuild `build/*.targets`, so it runs no package code — only `dotnet publish` + // (jailed below) does — so it runs unsandboxed. let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) @@ -384,31 +390,82 @@ async fn build_cs_proj( } } - let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str()); - build_cs_cmd - .current_dir(job_dir) - .env_clear() - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) - .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") - .env("DOTNET_NOLOGO", "true") - .env("MSBUILDDISABLENODEREUSE", "1") - .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) - .args(vec![ - "publish", - "--configuration", - "Release", - "-o", - &format!("{job_dir}/out"), - "--no-self-contained", - "-p:PublishSingleFile=true", - "-p:IncludeNativeLibrariesForSelfExtract=true", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let out_dir = format!("{job_dir}/out"); + let publish_args = vec![ + "publish", + "--configuration", + "Release", + "-o", + out_dir.as_str(), + "--no-self-contained", + "-p:PublishSingleFile=true", + "-p:IncludeNativeLibrariesForSelfExtract=true", + ]; + // `dotnet publish` imports and runs the MSBuild targets/tasks of restored NuGet + // packages (a package's build/*.targets execute at build, not at restore), i.e. + // arbitrary package code. Confine it under nsjail when sandboxing is on, the same + // way the run step is; HOME is set to /tmp by the config, keep_env passes the + // explicit dotnet/proxy vars, and the job dir + shared nuget cache are writable. + // Without sandboxing, publish runs directly with no isolation. + // mut is required by the #[cfg(windows)] env block below; unused on unix. + #[allow(unused_mut)] + let mut build_cs_cmd = if is_sandboxing_enabled() { + std::fs::create_dir_all(&*CSHARP_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + write_file( + job_dir, + "build.config.proto", + NSJAIL_CONFIG_BUILD_CSHARP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{CSHARP_CACHE_DIR}", &*CSHARP_CACHE_DIR) + .replace("{DOTNET_ROOT}", DOTNET_ROOT.as_str()) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + let mut nsjail_args = vec!["--config", "build.config.proto", "--", DOTNET_PATH.as_str()]; + nsjail_args.extend(publish_args.iter().copied()); + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + } else { + let mut cmd = Command::new(DOTNET_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .args(publish_args.clone()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + }; #[cfg(windows)] build_cs_cmd @@ -433,14 +490,20 @@ async fn build_cs_proj( .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), ); - let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str(), true).await?; + let build_cs_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + DOTNET_PATH.as_str() + }; + let build_cs_process = start_child_process(build_cs_cmd, build_cs_executable, true).await?; handle_child( job_id, conn, mem_peak, canceled_by, build_cs_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "dotnet publish", diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 22a2bf4e8b..c25ad82a83 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -210,6 +210,53 @@ pub async fn handle_child( } } + impl KillReason { + /// Reports a kill once the process is gone: a span event on the enclosing span and, + /// through the logs bridge, a log record carrying the job's log context. + fn emit_canceled_event( + &self, + job_id: &Uuid, + timeout_reason: &str, + signal: &'static str, + detected_at: chrono::DateTime, + ) { + // `force` is only known for a soft cancel: a job completed under the worker + // may have been force canceled or completed by anything else server-side. + let (kill_reason, canceled_by, canceled_reason, force) = match self { + KillReason::TooManyLogs => return, + KillReason::Timeout { .. } => { + ("timeout", Some("timeout"), Some(timeout_reason), None) + } + KillReason::Cancelled(by) => ( + "cancelled", + by.as_ref().and_then(|x| x.username.as_deref()), + by.as_ref().and_then(|x| x.reason.as_deref()), + Some(false), + ), + KillReason::AlreadyCompleted => ("already_completed", None, None, None), + }; + let exited_at = chrono::Utc::now(); + let root_job = windmill_common::log_context::current_log_context() + .and_then(|c| c.root_job.clone()); + let rfc3339 = |t: chrono::DateTime| { + t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) + }; + tracing::info!( + job_id = %job_id, + root_job, + kill_reason, + canceled_by, + canceled_reason, + force, + kill_signal = signal, + cancel_detected_at = %rfc3339(detected_at), + process_exited_at = %rfc3339(exited_at), + kill_duration_ms = (exited_at - detected_at).num_milliseconds(), + "job.canceled" + ); + } + } + let (timeout_duration, timeout_warn_msg, timeout_source) = resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await; if let Some(msg) = timeout_warn_msg { @@ -232,6 +279,7 @@ pub async fn handle_child( UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted, }, }; + let detected_at = chrono::Utc::now(); tx.send(()).expect("rx should never be dropped"); drop(tx); @@ -268,59 +316,78 @@ pub async fn handle_child( } }; - #[allow(unused_variables)] - if let Some(id) = child.id() { - if *MAX_WAIT_FOR_SIGINT > 0 { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); + // Resolves once the process is reaped, to the last signal it was sent, or to `None` + // when the kill could not be confirmed. SIGINT and SIGTERM are only sent where the + // sends below are compiled in: elsewhere an exit during their wait is unsignaled. + let unix_signal = |s: &'static str| { + if cfg!(any(target_os = "linux", target_os = "macos")) { s } else { "none" } + }; + let kill = async { + #[allow(unused_variables)] + if let Some(id) = child.id() { + if *MAX_WAIT_FOR_SIGINT > 0 { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); - for _ in 0..*MAX_WAIT_FOR_SIGINT { + for _ in 0..*MAX_WAIT_FOR_SIGINT { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + set_reason.await; + return Ok(Some(unix_signal("SIGINT"))); } - sleep(Duration::from_secs(1)).await; } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); + if sigterm { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); + + for _ in 0..*MAX_WAIT_FOR_SIGTERM { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } + if child.try_wait().is_ok_and(|x| x.is_some()) { + set_reason.await; + return Ok(Some(unix_signal("SIGTERM"))); + } } } - if sigterm { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); - - for _ in 0..*MAX_WAIT_FOR_SIGTERM { - if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + #[cfg(windows)] + { + let pid_to_kill = child.id(); + let killed = match kill_process_tree(pid_to_kill).await { + Ok(_) => { + tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ); + true } - sleep(Duration::from_secs(1)).await; - } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); - } + Err(e) => { + tracing::error!("failed to kill process tree: {:?}", e); + false + } + }; + set_reason.await; + return Ok(killed.then_some("taskkill")); } - } - #[cfg(windows)] - { - let pid_to_kill = child.id(); - match kill_process_tree(pid_to_kill).await { - Ok(_) => tracing::debug!( - "successfully killed process tree with PID: {:?}", - pid_to_kill - ), - Err(e) => tracing::error!("failed to kill process tree: {:?}", e), - }; - set_reason.await; - return Ok(Err(kill_reason)); - } - #[cfg(unix)] - { - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; - kill.map(|()| Err(kill_reason)) + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; + kill.map(|()| Some("SIGKILL")) + } + }; + let signal: io::Result> = kill.await; + if let Some(signal) = signal? { + kill_reason.emit_canceled_event(&job_id, &timeout_reason, signal, detected_at); } + Ok(Err(kill_reason)) }; let mut stream_result = Vec::new(); @@ -957,9 +1024,9 @@ where let conn = conn.clone(); // No tick at t=0: a job that finishes within the first interval issues none of the - // ping/metric statements below. Cancels are picked up by the first job ping (500 ms in, - // 2 s on agent workers). Memory is still sampled at start, without SQL: it is the only - // `mem_peak` reading a short job gets, and the completed job row reports it. + // ping/metric statements below. Cancels are picked up by the next job ping. Memory is still + // sampled at start, without SQL: it is the only `mem_peak` reading a short job gets, + // and the completed job row reports it. let mut interval = interval_at(Instant::now() + update_job_interval, update_job_interval); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); *mem_peak = (*mem_peak).max(get_mem.next().await.unwrap_or(0)); @@ -1023,11 +1090,10 @@ where } } } - if matches!(conn, Connection::Http(_)) { - if i % 4 != 0 { - // only ping every 4th time (2s) on http agent mode - continue; - } + // An agent worker pays an HTTP round trip per ping, so it pings every 2 s for + // the first 10 s, then every 5 s. + if matches!(conn, Connection::Http(_)) && i % if i < 20 { 4 } else { 10 } != 0 { + continue; } let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); @@ -1074,8 +1140,8 @@ fn child_joined_output_stream( .take() .expect("child did not have a handle to stdout"); - let stdout = BufReader::new(stdout).lines(); - let stderr = BufReader::new(stderr).lines(); + let stdout = BufReader::new(stdout).split(b'\n'); + let stderr = BufReader::new(stderr).split(b'\n'); stream::select( lines_to_stream(stderr, true, job_id.clone(), w_id.clone()) .map(|l| l.map(|line| OutputLine { stderr: true, line })), @@ -1085,18 +1151,32 @@ fn child_joined_output_stream( } pub fn lines_to_stream( - mut lines: tokio::io::Lines, + mut lines: tokio::io::Split, stderr: bool, job_id: Uuid, w_id: String, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) - .poll_next_line(cx) - .map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id)) + .poll_next_segment(cx) + .map(|result| { + let result = result.map(|segment| segment.map(decode_output_line)); + process_streaming_log_lines(result, stderr, &job_id, &w_id) + }) }) } +/// Child output is not guaranteed to be UTF-8 (e.g. pwsh on Windows writes in the +/// console code page), so decode lossily: a strict decode errors out and ends log +/// capture for the rest of the job. Strips a trailing `\r` so CRLF output doesn't keep it. +pub(crate) fn decode_output_line(mut segment: Vec) -> String { + if segment.last() == Some(&b'\r') { + segment.pop(); + } + String::from_utf8(segment) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) +} + pub fn process_status( program: &str, status: ExitStatus, @@ -1143,4 +1223,33 @@ mod tests { let left = JobDeadline(Some(Instant::now() + Duration::from_secs(120))); assert!(matches!(left.remaining_secs(), Some(s) if (118..=120).contains(&s))); } + + // "Prüfe" in CP850/1252 is not UTF-8. A strict decode errors on that line and + // would stop log capture, dropping every line after it. + #[tokio::test] + async fn a_non_utf8_line_is_decoded_lossily_and_later_lines_still_arrive() { + let output: &[u8] = + b"start\r\nPr\x81fe Benutzer\nPr\xfcfe\nGr\xc3\xbc\xc3\x9fe\r\nafter\n\nlast"; + let lines: Vec = lines_to_stream( + BufReader::new(output).split(b'\n'), + false, + Uuid::nil(), + "w".into(), + ) + .map(|line| line.expect("reading from a byte slice cannot fail")) + .collect() + .await; + assert_eq!( + lines, + vec![ + "start", + "Pr\u{FFFD}fe Benutzer", + "Pr\u{FFFD}fe", + "Grüße", + "after", + "", + "last", + ] + ); + } } diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 03328b82cb..8378455ef4 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -17,6 +17,7 @@ use windmill_queue::MiniPulledJob; use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; +use crate::common::DEV_CONF_NSJAIL; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, @@ -24,11 +25,13 @@ use crate::{ resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + PHP_PATH, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto"); +const NSJAIL_CONFIG_LOCK_PHP_CONTENT: &str = include_str!("../nsjail/lock.php.config.proto"); lazy_static::lazy_static! { static ref RE: Regex = Regex::new(r"^//\s?(\S+)\s*$").unwrap(); @@ -199,15 +202,99 @@ pub async fn composer_install( write_file(job_dir, "composer.lock", lock)?; } - let mut child_cmd = Command::new(&*COMPOSER_PATH); - let args = vec!["install", "--no-dev", "--no-progress"]; - child_cmd - .current_dir(job_dir) - .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, &*COMPOSER_PATH, false).await?; + // `composer install` runs the plugin and script code of installed packages, so + // when sandboxing is on it is confined under nsjail like the run step. Inside the + // jail the environment is cleared down to what composer needs: PATH, COMPOSER_HOME + // (pointed at the mounted cache dir), the proxies, the operator-controlled + // WHITELIST_ENVS, and every other COMPOSER_* var from the worker env — Windmill has + // no managed composer-registry setting, so operators supply private Packagist auth + // and composer tuning (COMPOSER_AUTH, COMPOSER_PROCESS_TIMEOUT, …) through those. + // COMPOSER_CACHE_DIR is dropped so it can't point composer outside the jail, and a + // custom COMPOSER_CAFILE/COMPOSER_CAPATH is bind-mounted so TLS to private feeds + // keeps working. COMPOSER_PATH is Windmill-internal. Without sandboxing composer + // runs directly, with no isolation. + let composer_args = ["install", "--no-dev", "--no-progress"]; + let child_cmd = if is_sandboxing_enabled() { + // The cache dir is bind-mounted into the jail, so it must exist before + // nsjail builds the mount tree. + std::fs::create_dir_all(&*COMPOSER_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + // A custom CA bundle/dir lives outside the jail's mounts; bind it in so + // composer's TLS can read it (mandatory: false → skipped when the path is unset). + let ca_mount = ["COMPOSER_CAFILE", "COMPOSER_CAPATH"] + .iter() + .filter_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())) + .map(|p| { + format!( + "mount {{\n src: \"{p}\"\n dst: \"{p}\"\n is_bind: true\n mandatory: false\n}}\n" + ) + }) + .collect::(); + write_file( + job_dir, + "lock.config.proto", + NSJAIL_CONFIG_LOCK_PHP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{COMPOSER_CACHE_DIR}", &*COMPOSER_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("{CA_MOUNT}", &ca_mount) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + + let composer_passthrough_envs: Vec<(String, String)> = std::env::vars() + .filter(|(k, _)| { + k.starts_with("COMPOSER_") + && k != "COMPOSER_HOME" + && k != "COMPOSER_CACHE_DIR" + && k != "COMPOSER_PATH" + }) + .collect(); + + let mut nsjail_args = vec![ + "--config", + "lock.config.proto", + "--", + COMPOSER_PATH.as_str(), + ]; + nsjail_args.extend(composer_args); + + let mut child_cmd = Command::new(NSJAIL_PATH.as_str()); + child_cmd + .current_dir(job_dir) + .env_clear() + // HOME is set to /tmp by the nsjail config; keep_env passes the rest. + .env("PATH", PATH_ENV.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .envs(composer_passthrough_envs) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + } else { + let mut child_cmd = Command::new(&*COMPOSER_PATH); + child_cmd + .current_dir(job_dir) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(composer_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + }; + let child_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + COMPOSER_PATH.as_str() + }; + let child_process = start_child_process(child_cmd, child_executable, false).await?; handle_child( job_id, @@ -215,7 +302,8 @@ pub async fn composer_install( mem_peak, canceled_by, child_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "composer install", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3fce0fb938..7f2291099a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -116,6 +116,7 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver, + index_url: Option, + /// Caches filled through these URLs must not be shared with other workspaces. + ws_scoped: bool, +} + +impl PipIndexes { + async fn snapshot(w_id: &str) -> Self { + let overrides = crate::workspace_registry_overrides(w_id).await; + let ws_value = |key: &str| match overrides.get(key) { + Some(serde_json::Value::String(s)) => Some(s.clone()), + _ => None, + }; + // An empty value means unset, and an empty workspace override unsets the global one. + Self { + extra_index_url: ws_value("pip_extra_index_url") + .or(PIP_EXTRA_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + index_url: ws_value("pip_index_url") + .or(PIP_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + ws_scoped: !overrides.is_empty(), + } + } + + /// The `(extra index, index)` URLs to hand uv. Kept apart from [`Self::snapshot`]: it runs + /// `EPHEMERAL_TOKEN_CMD`, which a caller served from a cache has no reason to pay for. + async fn urls( + self, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> (Option, Option) { + ( + read_ee_registry( + self.extra_index_url, + "pip extra index url", + job_id, + w_id, + conn, + ) + .await + .map(handle_ephemeral_token), + read_ee_registry(self.index_url, "pip index url", job_id, w_id, conn) + .await + .map(handle_ephemeral_token), + ) + } +} + lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); @@ -2527,28 +2634,18 @@ pub async fn handle_python_reqs( job_id ); - let pip_indexes = ( - read_ee_registry_with_workspace_override( - PIP_EXTRA_INDEX_URL.read().await.clone(), - "pip_extra_index_url", - "pip extra index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - read_ee_registry_with_workspace_override( - PIP_INDEX_URL.read().await.clone(), - "pip_index_url", - "pip index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - ); + let pip_indexes = PipIndexes::snapshot(w_id).await; + let ws_scoped = pip_indexes.ws_scoped; + let pip_indexes = pip_indexes.urls(job_id, w_id, conn).await; + // The wheel dir is shared by every workspace on the worker, and by every worker through + // the object store, under a name that says nothing of the index it was filled from. A + // workspace with its own registries gets its own, as its resolution cache does. The id + // is hashed: it is not restricted to characters safe in a file name. + let ws_dir_suffix = if ws_scoped { + format!("-ws-{}", &calculate_hash(w_id)[..16]) + } else { + String::new() + }; // Cached paths let mut req_with_penv: Vec<(String, String)> = vec![]; @@ -2577,7 +2674,7 @@ pub async fn handle_python_reqs( let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( - "{py_prefix}/{}", + "{py_prefix}/{}{ws_dir_suffix}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(venv_p.clone() + "/.valid.windmill").await.is_ok() { diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index b4bb448b99..24b07eacb9 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -855,7 +855,7 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value { } }) } - "binary" => { + "binary" | "varbinary" => { // convert base64 to hex as expected by snowflake let bytes = engine::general_purpose::STANDARD .decode(arg_v.as_str().unwrap_or("")) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 80ca37acb8..d0d5d8b4e2 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -155,7 +155,7 @@ use crate::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, tally_unfinished_dependency_deploy, }, - worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full}, + worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full}, }; #[cfg(feature = "rust")] @@ -949,6 +949,22 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The registry overrides of a workspace, empty when it has none. +/// +/// One snapshot: registry settings reload under running jobs, so a caller whose cache key +/// has to agree with the registry values it used reads both from here rather than calling +/// [`workspace_registry_cache_suffix`] and a `read_ee_registry_*` helper separately. +pub async fn workspace_registry_overrides( + w_id: &str, +) -> std::collections::HashMap { + let registries = WORKSPACE_REGISTRIES.read().await; + registries + .as_ref() + .and_then(|m| m.get(w_id)) + .cloned() + .unwrap_or_default() +} + /// The name a build artifact is cached under, derived from `base` — the runnable's own /// cache-key input — and its inline modules. /// @@ -2005,6 +2021,7 @@ pub fn create_span_with_name( root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, + worker_group = %*WORKER_GROUP, hostname = field::Empty, tag = %arc_job.tag, language = field::Empty, @@ -2156,6 +2173,7 @@ pub fn log_context_for_job( job_id: Some(arc_job.id.to_string()), workspace_id: Some(arc_job.workspace_id.clone()), worker: Some(worker_name.to_string()), + worker_group: Some(WORKER_GROUP.clone()), tag: Some(arc_job.tag.clone()), job_kind: Some(arc_job.kind.as_str().to_string()), created_by: Some(arc_job.created_by.clone()), @@ -2695,7 +2713,7 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); let mut reported_ip = cached_ip(); - let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn) + let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn) .await .expect("initial ping could be sent"); @@ -3142,6 +3160,31 @@ pub async fn run_worker( let mut killpill_rx2 = killpill_rx.resubscribe(); + // The loop below only sees the killpill once the running job is done, which is too late + // for that job to learn its worker is going away. + { + let mut killpill_rx = killpill_rx.resubscribe(); + let conn = conn.clone(); + let worker_name = worker_name.clone(); + tokio::spawn(async move { + if matches!( + killpill_rx.recv().await, + Err(broadcast::error::RecvError::Closed) + ) { + return; + } + for attempt in 1..=3 { + match mark_worker_draining(&conn, &worker_name).await { + Ok(()) => return, + Err(e) => { + tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}"); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + }); + } + loop { let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst); if last_processing_duration_secs > 5 { @@ -3217,7 +3260,7 @@ pub async fn run_worker( // which costs at most the last job's id here: no job of this worker is in flight at this // point in the loop, and the next one refills them. if ip_just_resolved && conn.as_sql().is_none() { - if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await { + if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await { tracing::warn!( worker = %worker_name, hostname = %hostname, "failed to re-register with the resolved external IP: {e}" @@ -3406,6 +3449,10 @@ pub async fn run_worker( tokio::time::sleep(Duration::from_millis(200)).await; continue; } + } else if WORKER_CONFIG.load().paused { + // Checked after the same-worker channel: a flow this worker already runs + // still needs it for its remaining same-worker steps. + Ok(None) } else { match &conn { Connection::Sql(db) => { diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d2cb033084..6d76d0d771 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -192,6 +192,7 @@ pub async fn update_flow_status_after_job_completion( flow_job_duration, stop_early_override, has_triggered_error_handler: false, + recovered_by_child: false, }; let mut step_failure = step_failure; loop { @@ -211,6 +212,7 @@ pub async fn update_flow_status_after_job_completion( worker_dir, rec.stop_early_override, rec.has_triggered_error_handler, + rec.recovered_by_child, worker_name, job_completed_tx.clone(), flow_runners.clone(), @@ -240,6 +242,7 @@ pub async fn update_flow_status_after_job_completion( worker_dir, rec.stop_early_override, rec.has_triggered_error_handler, + false, worker_name, job_completed_tx.clone(), flow_runners.clone(), @@ -293,6 +296,10 @@ pub struct RecUpdateFlowStatusAfterJobCompletion { flow_job_duration: Option, stop_early_override: Option, has_triggered_error_handler: bool, + /// The failed child's error handler returned `recover: true`. The parent still handles the + /// failure as it would any other, so the same steps run; only a root flow that this failure + /// ends turns green. + recovered_by_child: bool, } #[derive(Deserialize)] @@ -427,6 +434,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_dir: &str, stop_early_override: Option, has_triggered_error_handler: bool, + mut recovered_by_child: bool, worker_name: &str, job_completed_tx: JobCompletedSender, flow_runners: Option>, @@ -449,6 +457,7 @@ pub async fn update_flow_status_after_job_completion_internal( skip_if_stop_early, nresult, is_failure_step, + failure_step_recovers, _cleanup_module, chat_ai_info, ) = { @@ -1096,6 +1105,9 @@ pub async fn update_flow_status_after_job_completion_internal( } else { has_triggered_error_handler = false; } + // Recovered iterations are stored as successes, so a failed parallel + // node failed on an iteration that did not recover. + recovered_by_child = false; (success, Some(new_status)) } else { add_time!(bench, "handle parallel flow start"); @@ -1823,6 +1835,13 @@ pub async fn update_flow_status_after_job_completion_internal( _ => false, }; + let failure_step_recovers = is_failure_step && result_has_recover_true(nresult.clone()); + // An error this flow raises on its own (a stop predicate's error message) is a new + // failure that no error handler recovered. + if stop_early_err_msg.is_some() { + recovered_by_child = false; + } + let chat_ai_info = ChatAiInfo { chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false), conversation_id: old_status.memory_id, @@ -1837,6 +1856,7 @@ pub async fn update_flow_status_after_job_completion_internal( skip_if_stop_early, nresult, is_failure_step, + failure_step_recovers, old_status.cleanup_module, chat_ai_info, ) @@ -1844,6 +1864,10 @@ pub async fn update_flow_status_after_job_completion_internal( let flow_job = Arc::new(flow_job); + // An inner flow that ends on a recovered failure stays failed, as parents read their + // children's stored status (a parallel node counts its iterations' statuses), so only the + // root flow turns green. + let ends_root = !(flow_job.is_flow_step() && flow_job.parent_job.is_some()); let done = if !should_continue_flow { { let logs = if flow_job.is_canceled() { @@ -1851,7 +1875,9 @@ pub async fn update_flow_status_after_job_completion_internal( } else if stop_early { format!("Flow job stopped early because of a stop early predicate returning true\n") } else if is_failure_step { - format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, result_has_recover_true(nresult.clone())) + format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, failure_step_recovers) + } else if !success && recovered_by_child && ends_root { + format!("Flow job completed with success: the step that failed had its error recovered by the error handler (recover: true)\n") } else { format!( "Flow job completed with {}\n", @@ -2048,7 +2074,11 @@ pub async fn update_flow_status_after_job_completion_internal( .await; } - let success = success && (!is_failure_step || result_has_recover_true(nresult.clone())); + let success = if is_failure_step { + success && failure_step_recovers + } else { + success || (recovered_by_child && ends_root) + }; add_time!(bench, "flow status update 1"); @@ -2149,6 +2179,8 @@ pub async fn update_flow_status_after_job_completion_internal( .await; // override prior child's success so the parent learns this flow failed success = false; + // a new failure, which no error handler recovered + recovered_by_child = false; true } Ok(_) => false, @@ -2178,6 +2210,7 @@ pub async fn update_flow_status_after_job_completion_internal( None }, has_triggered_error_handler: has_triggered_error_handler || is_failure_step, + recovered_by_child: failure_step_recovers || (recovered_by_child && !success), }, )); } diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 3a313e9360..cabf004f4d 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -7,9 +7,9 @@ use windmill_common::{ external_ip::UNKNOWN_IP, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, - insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, - update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED, - WORKER_CONFIG, WORKER_GROUP, + insert_ping_query, set_worker_draining_query, update_job_ping_query, + update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping, + PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, DB, }; @@ -164,6 +164,7 @@ async fn update_worker_ping_full_inner( wm_memory_usage: get_windmill_memory_usage(), job_isolation: None, native_mode: Some(native_mode), + draining: None, ping_type: PingType::MainLoop, }, ) @@ -179,6 +180,7 @@ pub async fn insert_ping( worker_instance: &str, worker_name: &str, ip: Option<&str>, + reset_draining: bool, db: &Connection, ) -> anyhow::Result { let (tags, dw, dws, native_mode) = { @@ -224,6 +226,7 @@ pub async fn insert_ping( memory, job_isolation, native_mode, + reset_draining.then_some(false), db, ) .await; @@ -256,6 +259,7 @@ pub async fn insert_ping( wm_memory_usage: get_windmill_memory_usage(), job_isolation, native_mode: Some(native_mode), + draining: reset_draining.then_some(false), ping_type: PingType::Initial, }, ) @@ -314,6 +318,7 @@ pub async fn update_worker_ping_from_job( &Ping { last_job_executed: Some(job_id.clone()), last_job_workspace_id: Some(w_id.to_string()), + draining: None, ping_type: PingType::Job, worker_instance: None, ip: None, @@ -342,6 +347,45 @@ pub async fn update_worker_ping_from_job( Ok(()) } +/// Publishes on the worker's ping row that it has received its killpill, so the job it is +/// still running can find out and exit early. +pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> { + match conn { + Connection::Sql(db) => set_worker_draining_query(worker_name, db).await, + Connection::Http(client) => { + client + .post::( + UPDATE_PING_URL, + None, + &Ping { + last_job_executed: None, + last_job_workspace_id: None, + draining: None, + ping_type: PingType::Draining, + worker_instance: None, + ip: None, + tags: None, + dw: None, + dws: None, + version: None, + vcpus: None, + memory: None, + memory_usage: None, + wm_memory_usage: None, + jobs_executed: None, + occupancy_rate: None, + occupancy_rate_15s: None, + occupancy_rate_5m: None, + occupancy_rate_30m: None, + job_isolation: None, + native_mode: None, + }, + ) + .await + } + } +} + pub async fn ping_job_status( conn: &Connection, job_id: &Uuid, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2b5215da99..df2044d419 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.826.0"; +export const VERSION = "v1.828.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index fcc48449cf..c6691e0048 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.826.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.826.0", + "version": "1.828.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 33b6c43fa3..a119b8645d 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.826.0", + "version": "1.828.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts index 87d40f161b..b011965ae8 100644 --- a/cli/src/commands/app/new.ts +++ b/cli/src/commands/app/new.ts @@ -908,7 +908,7 @@ const command = new Command() ) .option( "--schema ", - "Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist." + "Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it." ) .option( "--overwrite", diff --git a/cli/src/commands/user/user.ts b/cli/src/commands/user/user.ts index b5ed0e3196..99bea072a8 100644 --- a/cli/src/commands/user/user.ts +++ b/cli/src/commands/user/user.ts @@ -1,6 +1,8 @@ import { writeFile } from "node:fs/promises"; import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import { GlobalOptions, isSuperset, @@ -69,6 +71,32 @@ async function list(opts: GlobalOptions) { .render(); } +async function listWorkspaceUsers(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const users = await wmill.listUsers({ workspace: workspace.workspaceId }); + + if (opts.json) { + console.log(JSON.stringify(users)); + } else { + new Table() + .header(["username", "email", "role"]) + .padding(2) + .border(true) + .body( + users.map((u) => [ + u.username, + u.email, + u.is_admin ? "admin" : u.operator ? "operator" : "developer", + ]) + ) + .render(); + } +} + function rdString() { return Math.random().toString(36).slice(2, 7); } @@ -518,6 +546,9 @@ export async function pushInstanceGroups( const command = new Command() .description("user related commands") .action(list as any) + .command("list", "List the users of the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(listWorkspaceUsers as any) .command("add", "Create a user") .arguments(" [password:string]") .option("--superadmin", "Specify to make the new user superadmin.") diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 6b59042168..860a3ddede 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.826.0"; +export const VERSION = "1.828.0"; diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 6060f6b8a5..dc2a2f8d84 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -90,7 +90,7 @@ A data pipeline is NOT a flow: it is a set of independent scripts in one folder, ## Raw App Development You MUST use the \`raw-app\` skill to create or modify raw apps. -When a new app needs to be created, YOU run \`wmill app new\` yourself with \`--summary\`, \`--path\`, and \`--framework\` flags (and any other relevant flags). Do NOT ask the user to run it. If you don't have the values for those flags, ask the user via \`AskUserQuestion\` (one call, all missing fields) — never invent them. See the \`raw-app\` skill for the full procedure. +When a new app needs to be created, YOU run \`wmill app new\` yourself with \`--summary\`, \`--path\`, and \`--framework\` flags (and any other relevant flags). Do NOT ask the user to run it. If you don't have a summary or a path, ask the user via \`AskUserQuestion\` (one call, all missing fields) — never invent them. The framework is not asked: \`react19\` unless the user or the project's existing apps say otherwise. An app that stores data also needs its data table and schema settled with the user first. See the \`raw-app\` skill for the full procedure. ## Triggers diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 86a1395861..eb5d46ef57 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -767,21 +767,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -837,6 +839,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1600,21 +1610,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1670,6 +1682,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2539,21 +2559,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -2609,6 +2631,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -4434,10 +4464,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -4610,6 +4644,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -5707,6 +5753,7 @@ These are strict Windmill schema rules. Follow them exactly. - \`preprocessor_module\` and \`failure_module\` only support \`script\` or \`rawscript\` - \`preprocessor_module\` runs before normal modules and cannot reference \`results.*\` - \`failure_module\` can use the \`error\` object with \`error.message\`, \`error.step_id\`, \`error.name\`, and \`error.stack\` +- A flow whose \`failure_module\` runs still ends as failed, unless the handler returns an object with \`recover: true\`: the flow then ends as a success. This also holds when the failing step is inside a loop, a branch or a subflow. \`recover\` never changes which steps run; whether a loop or branch carries on past a failure is still decided by its own settings (\`skip_failures\`, \`continue_on_error\`) Correct shape: @@ -5864,7 +5911,7 @@ names, so neither is name-checked at all — leave those summaries as they are. ## AI Decision Modules -An \`aidecision\` module asks a decision model (TypeSafe's Jev or Cloudflare's Clef) typed questions about a \`state\` and +An \`aidecision\` module asks a decision model (TypeSafe's Jev, Cloudflare's Clef or OpenAI's Decisions API) typed questions about a \`state\` and answers each with calibrated probabilities instead of text. Prefer it over an \`aiagent\` when the step is a judgment (classify, route, score or a yes/no check) that needs no tools and no free text: it is faster, cheaper, and its answers have a fixed shape. @@ -5902,8 +5949,9 @@ is faster, cheaper, and its answers have a fixed shape. } \`\`\` -- \`provider.kind\` is \`typesafe\` (\`model\` \`jev-latest\` unless a version is pinned) or \`cloudflare\` - (\`model\` \`clef\`, or \`clef-flash\` for faster answers); the resource is of that same type +- \`provider.kind\` is \`typesafe\` (\`model\` \`jev-latest\` unless a version is pinned), \`cloudflare\` + (\`model\` \`clef\`, or \`clef-flash\` for faster answers) or \`openai\` (\`model\` \`gpt-6-luna\`, never a + chat model); the resource is of that same type - \`state\` is the content to evaluate: usually a text, such as the message to classify. To combine several values, pass an object with descriptive keys holding only what the questions need (\`({ message: flow_input.message, plan: results.get_account.plan })\`); an array of strings also works @@ -5917,6 +5965,12 @@ is faster, cheaper, and its answers have a fixed shape. - The result is \`{ output, model, usage }\` with \`output\` keyed by question name, so a later step reads \`results.triage.output.intent.choice\`, \`results.triage.output.urgency.score\` or \`results.triage.output.angry.noul > 0.7\` +- On \`openai\` the same \`questions\` are translated for its Decisions API and the answers read the + same way (\`choice\`, \`score\`, \`noul\`), except that \`probabilities\` is a list of + \`{ value, probability }\` (plus \`label\` on a score) and a score has no \`legend\`. A question OpenAI + refuses to answer fails the step. A \`state\` that is + an array of user messages (\`[{ role: "user", content: [{ type: "input_text", text }, { type: "input_image", image_url: "data:image/png;base64,..." }] }]\`) + is sent as is, which is how an image is evaluated; any other object or array is sent as JSON text ### Branching on the Answers @@ -5961,6 +6015,10 @@ can call one, not an agent used as a tool. - \`results.step_id.property\` - Access specific property from a previous step output only when that step result is in scope - \`flow_input.iter.value\` - Current iteration value inside a \`forloopflow\`; in a \`whileloopflow\` it is just the iteration index (a plain number, same as \`flow_input.iter.index\`) - \`flow_input.iter.index\` - Current loop index when inside a loop (\`forloopflow\` or \`whileloopflow\`) +- After a suspend step (approval), the next step's javascript expressions also get: + - \`resume\` - the last resume payload (the resume form values); with \`continue_on_disapprove_timeout\`, a disapproval or timeout yields \`{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }\` instead + - \`resumes\` - every resume payload, in arrival order + - \`approvers\` - a plain array of strings, one per resume: for a resume through the \`resume\`/\`cancel\` URL from \`getResumeUrls(approver)\`, the \`approver\` passed there if any; otherwise (including the \`approvalPage\` link) the resuming user's username, else \`"anonymous"\`, e.g. \`approvers[0]\`. It is NOT the \`{ resume_id, approver }\` objects stored in the flow status, so \`approvers[0].approver\` is always undefined ## Loop Structure Rules @@ -6272,7 +6330,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, or Cloudflare's\\nJev-compatible Clef) that answers typed questions about a state with calibrated\\nprobabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} `, "raw-app": `--- name: raw-app @@ -6287,47 +6345,53 @@ This guide covers raw apps from the terminal: scaffolding via \`wmill app new\`, **You — the AI agent — create the app yourself by running \`wmill app new\` with the right flags. Do NOT tell the user to "run \`wmill app new\` and follow the prompts" or wait for them to do it.** The bare \`wmill app new\` is an interactive wizard that hangs waiting for stdin in any non-TTY context (which includes you). Always pass flags. -### Step 1 — Gather the three required values by asking the user +### Step 1 — Settle the setup -You need three things to run the command: +You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. \`f/folder/my_app\` or \`u/username/my_app\` 3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\`, \`vue\` -If the user's request did not supply *every* one of these explicitly, ask. Do not guess values, do not invent paths, do not pick a framework on the user's behalf, do not "just use react19 because it's the default". +**summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. -Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group all missing fields into a single round-trip so the user answers them at once: +**framework**: never ask. Use the one the user named, otherwise the one the project's existing apps use, otherwise \`react19\` (see "Starting a new app" in the authoring guide). + +**data setup**: when the app has to store data and the user did not say where, ask which data table, which schema and which tables, as "Data setup" in the authoring guide lays out. \`wmill datatable list\` gives the data tables to offer. + +Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group everything missing into a single round-trip so the user answers at once: -- For \`framework\` — multiple-choice with the four allowed values; mark \`react19\` as \`(Recommended)\` and put it first. - For \`summary\` and \`path\` — provide one or two example values as multiple-choice options (the user can pick "Other" to type a free-form answer). +- For the data setup — the choices the authoring guide lists, the default first. -Only proceed once you have concrete values for all three. If the user replies with something ambiguous, ask again rather than guessing. +Only proceed once every value is concrete. If the user replies with something ambiguous, ask again rather than guessing. ### Step 2 — Run the command yourself -Once you have summary + path + framework, run it: +Once the setup is settled, run it: \`\`\`bash wmill app new \\ --summary "Customer dashboard" \\ --path f/sales/dashboard \\ - --framework react19 + --framework react19 \\ + --datatable main \\ + --schema app1 \`\`\` -That's the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of \`--summary\`/\`--path\`/\`--framework\` puts the command in non-interactive mode. +\`--summary\`, \`--path\` and \`--framework\` are the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of them puts the command in non-interactive mode, so the data setup only reaches the app through the flags below. ### Optional flags -Layer these in only when the user asked for them: - | Flag | When to add it | |---|---| -| \`--datatable \` | The user wants this app wired to a specific Windmill datatable. Without it, the app is created with no datatable. | -| \`--schema \` | Together with \`--datatable\`. Creates the schema with \`CREATE SCHEMA IF NOT EXISTS\` if it doesn't already exist. | +| \`--datatable \` | The app stores data: the data table settled in step 1. Without it, the app is created with no datatable. | +| \`--schema \` | Together with \`--datatable\`, when the app's tables go in a schema, new or existing. If the schema doesn't exist yet, writes a \`sql_to_apply/\` migration with \`CREATE SCHEMA IF NOT EXISTS\`, applied like any other migration (see "SQL Migrations" below). Leave it out for \`public\`. | | \`--overwrite\` | The target directory already exists and the user said it's OK to replace. Without it, non-interactive mode aborts with an error so you don't clobber existing work. | | \`--no-open-in-desktop\` | Already implied in non-interactive mode; only needed if you're somehow running interactively. | +Existing tables the app reuses have no flag: list them under \`data.tables\` in \`raw_app.yaml\` (see "Data tables" below). + ### Step 3 — Offer the visual preview After \`wmill app new\` and any initial edits to \`App.tsx\` / \`index.tsx\`, **offer** to open the visual preview as a one-sentence next step (e.g. "Want me to open the visual preview?"). Don't auto-open — opening the dev page has side effects (browser window, possibly a \`launch.json\` entry when an embedded preview tool is in play) the user should consent to. @@ -6338,8 +6402,9 @@ For apps the preview command runs from the app folder (\`cd __raw_app - ❌ Running \`wmill app new\` with no flags (the prompt will hang). - ❌ Telling the user to "run \`wmill app new\` and follow the prompts" — that's a step backwards from what you can do directly. -- ❌ Inventing a path/summary/framework instead of asking the user. -- ❌ Defaulting to \`react19\` because the user didn't say — even sensible defaults must be confirmed. +- ❌ Inventing a path or summary instead of asking the user. +- ❌ Asking which framework to use — \`react19\` is the default when nothing says otherwise. +- ❌ Creating a schema or tables for a new app before the user has said where its data goes. - ❌ Passing \`--overwrite\` automatically when the directory exists — confirm with the user first. ### Interactive (only when a human is at the terminal) @@ -6526,6 +6591,38 @@ A raw app has three logical parts: - **Backend runnables** — server-side scripts the frontend calls, each addressed by a unique key. - **Data** — optional whitelisted datatables (managed PostgreSQL) that the backend runnables can query. The frontend never queries the database directly; backend runnables are the only bridge. +## Starting a new app + +A new app gets the setup Windmill's own new-app dialog gives one: a framework, then where its data lives. + +### Framework + +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. + +### Data setup + +When a new app has to store data and the user did not say where, **ask before creating anything**: no app, no schema, no table. Where the data lives is the user's decision, and tables created in the wrong place stay there. Look up the workspace's data tables first so every choice you offer exists, then ask, as proposed answers rather than open questions: + +1. **Data table**: which one to use, the default first (\`main\` when it exists, otherwise the first listed). Skip this one when the workspace has a single data table. +2. **Schema**: where the app's new tables go. + - A **new schema**, the usual choice, offered first. Suggest the first unused name among \`app1\`, \`app2\`, … + - An **existing schema** of that data table, by name. + - **None**: the tables go in \`public\`. +3. **Tables**: only when that data table already holds tables the app could use, whether to **reuse** them (name them) or **create new ones**. + +Do not ask when: + +- the user already said where the data goes. A part they left out takes its default above: the default data table, a new \`appN\` schema, new tables; +- the app, or its saved draft, already carries a data config (\`data.datatable\`, \`data.schema\` or \`data.tables\`). That config is the answer; +- the app stores nothing. + +With no data table in the workspace there is nothing to choose: say that one has to be configured in the workspace settings before the app can store data. + +A question left unanswered or dismissed is not a go-ahead for the defaults: stop there and say what is still needed. + +Then build what was chosen: the data table and schema become the app's \`data.datatable\` and \`data.schema\`, the tables it uses, reused or new, go in \`data.tables\`, and a new schema is created (\`CREATE SCHEMA IF NOT EXISTS\`) before the tables that go in it. +\`wmill app new --datatable --schema \` records both and, for a schema that does not exist yet, writes the migration in \`sql_to_apply/\` that creates it: nothing exists in the database until \`wmill app dev\` applies that migration, so the app's table migrations come after it. + ## Frontend ### Entrypoint @@ -7818,10 +7915,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -7865,7 +8000,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -7980,7 +8115,7 @@ app related commands - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - - \`--schema \` - Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist. + - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. - \`--no-open-in-desktop\` - Do not prompt to open the new app in Claude Desktop. - \`app generate-agents [app_folder:string]\` - regenerate AGENTS.md and DATATABLES.md from remote workspace @@ -8664,6 +8799,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. @@ -9014,7 +9151,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9025,7 +9163,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9113,7 +9252,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9124,7 +9264,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9197,7 +9338,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9208,7 +9350,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9318,7 +9461,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9329,7 +9473,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9481,7 +9626,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9492,7 +9638,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9650,7 +9797,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9661,7 +9809,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9776,7 +9925,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9787,7 +9937,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9874,7 +10025,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9885,7 +10037,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9966,7 +10119,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9977,7 +10131,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10064,7 +10219,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10075,7 +10231,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10215,7 +10372,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10226,7 +10384,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10416,7 +10575,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10427,7 +10587,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/docs/pipeline-local-dev.md b/docs/pipeline-local-dev.md index e05acec9f0..fb8d49b296 100644 --- a/docs/pipeline-local-dev.md +++ b/docs/pipeline-local-dev.md @@ -69,7 +69,7 @@ makes the client own the whole cascade so the backend dispatcher never double-fi - `frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts` — reusable run primitives (`makeLaunch`, `makeWaitJobTerminal`, `runDownstreamCascade`, `runBoundedCascade`) wrapping `cascadeOrchestrator` + `graphTraversal`. -- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/pipeline_dev/+page.svelte`. +- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/(styled)/pipeline_dev/+page.svelte`. **Modified** - `cli/src/commands/pipeline/pipeline.ts` — `--local` on `show`/`run`; `show` split into graph diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e8ea3dece4..28ecbebf42 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.828.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 259a83b5da..cc42f5424a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.828.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/app.html b/frontend/src/app.html index aad967fd91..3c9e232e0f 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -12,6 +12,17 @@ fill: #3b82f6; } + /* Public app pages render their shell before app.css loads, if at all. */ + html[data-wm-public-app] #svelte-global-loader { + display: none; + } + html[data-wm-public-app] body { + margin: 0; + } + html[data-wm-public-app='dark'] body { + background-color: #3b4252; + } + /* Background color for dark mode */ @media (prefers-color-scheme: dark) { #mainbg { @@ -123,6 +134,64 @@ } })() + %sveltekit.head% diff --git a/frontend/src/lib/components/AIProviderPicker.svelte b/frontend/src/lib/components/AIProviderPicker.svelte index 6369d0abb1..67afad9be6 100644 --- a/frontend/src/lib/components/AIProviderPicker.svelte +++ b/frontend/src/lib/components/AIProviderPicker.svelte @@ -24,8 +24,8 @@ * other than the one being navigated. Resources and the models read off them are per * workspace, so without it this offers what the wrong one holds. */ workspace?: string | undefined - /** Offer the decision providers (TypeSafe, Cloudflare) in place of the chat ones, for an AI - * decision. */ + /** Offer the decision providers (TypeSafe, Cloudflare, OpenAI) and their decision models in + * place of the chat ones, for an AI decision. */ decision?: boolean } @@ -84,6 +84,15 @@ return r }) + // A decision provider's models are the ones Windmill knows: none has a listing of its decision + // models, and an OpenAI resource's lists its chat models. + function defaultModels(provider: AIProvider): string[] { + return decision + ? ((DECISION_AI_PROVIDERS as Record)[provider] + ?.defaultModels ?? []) + : aiProviderDetails(provider).defaultModels + } + async function loadModels(signal?: AbortSignal) { const provider = value?.kind const resourceValue = value?.resource @@ -92,6 +101,11 @@ if (!provider || !resourcePath) { return } + if (decision) { + availableModels = defaultModels(provider) + loading = false + return + } loading = true const cacheKey = `${effectiveWorkspace}:${provider}:${resourcePath}` @@ -113,8 +127,7 @@ return } // Fall back to default models for this provider - const defaultModels = aiProviderDetails(provider).defaultModels - availableModels = defaultModels + availableModels = defaultModels(provider) } finally { if (!signal?.aborted) { loading = false @@ -178,8 +191,7 @@ if (provider && resourcePath) { loadModels(abortController.signal) } else { - const defaultModels = provider ? aiProviderDetails(provider).defaultModels : [] - availableModels = defaultModels + availableModels = provider ? defaultModels(provider) : [] loading = false } diff --git a/frontend/src/lib/components/AutosaveIndicator.svelte b/frontend/src/lib/components/AutosaveIndicator.svelte index 76dc030c58..1e318ed9b2 100644 --- a/frontend/src/lib/components/AutosaveIndicator.svelte +++ b/frontend/src/lib/components/AutosaveIndicator.svelte @@ -142,7 +142,13 @@ let hintTimer: ReturnType | undefined const kindLabel = $derived( - itemKind === 'flow' ? 'flow' : itemKind === 'app' || itemKind === 'raw_app' ? 'app' : 'script' + itemKind === 'flow' + ? 'flow' + : itemKind === 'app' || itemKind === 'raw_app' + ? 'app' + : itemKind === 'data_pipeline' + ? 'pipeline' + : 'script' ) // Each false → true transition fires a hint (others > loaded). Seeded to diff --git a/frontend/src/lib/components/BreadcrumbItemContent.svelte b/frontend/src/lib/components/BreadcrumbItemContent.svelte new file mode 100644 index 0000000000..9e945586c7 --- /dev/null +++ b/frontend/src/lib/components/BreadcrumbItemContent.svelte @@ -0,0 +1,16 @@ + + + +{@render icon?.()} +{label} diff --git a/frontend/src/lib/components/BreadcrumbSegment.svelte b/frontend/src/lib/components/BreadcrumbSegment.svelte index 8a266e4fc4..4d55675f26 100644 --- a/frontend/src/lib/components/BreadcrumbSegment.svelte +++ b/frontend/src/lib/components/BreadcrumbSegment.svelte @@ -6,7 +6,7 @@ state — switching popovers relies on melt-ui's `closeOnOtherPopoverOpen` to close siblings. -->
import { AlertTriangle, - ArrowDownRight, + ArrowDown, ArrowRight, - ArrowUpRight, + ArrowUp, Building, CircleCheck, CircleX, @@ -16,7 +16,7 @@ RefreshCw, UserPlus } from 'lucide-svelte' - import { untrack } from 'svelte' + import { onDestroy, untrack } from 'svelte' import type { CiTestResult } from '$lib/gen' import { Alert, Badge } from './common' import { @@ -97,6 +97,8 @@ onRetry?: () => void onSelectTarget?: (target: string) => void comparison: WorkspaceComparison | undefined + /** The comparison is still in flight, so an empty list says nothing yet. */ + comparisonLoading?: boolean /** Set when the comparison request failed (e.g. not an admin of the target). */ comparisonError?: string /** Initial merge direction; lets the page restore the chosen direction when @@ -144,6 +146,7 @@ onRetry, onSelectTarget, comparison, + comparisonLoading = false, comparisonError, initialMergeIntoParent = true, deployCount = 0, @@ -486,6 +489,12 @@ // Summary cache: stores summaries from both workspaces type SummaryCache = Record let summaryCache = $state({}) + // What has already been asked for. Deliberately not `$state`: the fetchers below run inside an + // effect, and a guard that reads reactive state makes that effect depend on what it writes — + // which is a loop. A plain Set is also what tells "asked, and the answer was nothing" from + // "never asked", which a value map cannot when the value for both is `undefined`. + const summariesRequested = new Set() + const onBehalfOfRequested = new Set() // On-behalf-of tracking for flows and scripts // Source workspace on_behalf_of emails (keyed by workspace/kind:path) @@ -499,51 +508,101 @@ return `${diff.kind}:${diff.path}` } + /** The two things a row needs from an item — the name it shows and who it runs as — are fields + * of the same object, so it is read once per item per workspace and awaited by both loops. + * + * `SHARED_READ_KINDS` is the kinds that carry both. A folder has a summary and no identity; a + * schedule or a trigger has an identity read from its own endpoint, which `getOnBehalfOf` + * dispatches. */ + const SHARED_READ_KINDS = ['script', 'flow', 'app', 'raw_app'] + type ItemRead = { summary?: string; onBehalfOf?: string } + const itemReads = new Map>() + + function readItem(kind: string, path: string, workspace: string): Promise { + const key = `${workspace}/${kind}:${path}` + let pending = itemReads.get(key) + if (!pending) { + // A failure is not an answer, so it is not kept: held, it would read as "this item has + // no identity", which is what decides whether the deploy offers to preserve one — and + // an item deployed without it silently becomes the deployer's. Each caller swallows its + // own, as both did before they shared a read. + pending = loadItem(kind, path, workspace).catch((error) => { + itemReads.delete(key) + throw error + }) + itemReads.set(key, pending) + } + return pending + } + + async function loadItem(kind: string, path: string, workspace: string): Promise { + if (kind === 'script') { + const script = await ScriptService.getScriptByPath({ workspace, path }) + return { summary: script.summary, onBehalfOf: script.on_behalf_of_email } + } else if (kind === 'flow') { + const flow = await FlowService.getFlowByPath({ workspace, path }) + return { summary: flow.summary, onBehalfOf: flow.on_behalf_of_email } + } else if (kind === 'app' || kind === 'raw_app') { + const app = await AppService.getAppByPath({ workspace, path }) + return { summary: app.summary, onBehalfOf: app.policy?.on_behalf_of_email } + } else if (kind === 'folder') { + const folder = await FolderService.getFolder({ workspace, name: path.replace(/^f\//, '') }) + return { summary: folder.summary } + } + return {} + } + async function fetchSummary( kind: string, path: string, workspace: string ): Promise { try { - if (kind === 'script') { - const script = await ScriptService.getScriptByPath({ workspace, path }) - return script.summary - } else if (kind === 'flow') { - const flow = await FlowService.getFlowByPath({ workspace, path }) - return flow.summary - } else if (kind === 'app' || kind === 'raw_app') { - const app = await AppService.getAppByPath({ workspace, path }) - return app.summary - } else if (kind === 'folder') { - const folder = await FolderService.getFolder({ workspace, name: path.replace(/^f\//, '') }) - return folder.summary - } + return (await readItem(kind, path, workspace)).summary } catch (error) { console.error(`Failed to fetch summary for ${kind}:${path}`, error) + return undefined } - return undefined } + // Both loops below run one item at a time and outlive the mount that started them: the page + // remounts this component when either side of the pair changes, and whatever was in flight then + // still resolves. It must not be stored — the summary key is kind and path, which the other + // pair spells the same way — and the loop must not keep asking for a pair nobody is looking at. + let alive = true + onDestroy(() => { + alive = false + }) + async function fetchSummaries(diffs: WorkspaceItemDiff[]) { // Only fetch summaries for scripts, flows, and apps const itemsToFetch = diffs.filter((diff) => ['script', 'flow', 'app', 'raw_app', 'folder'].includes(diff.kind) ) + // Read once, so every request in this pass names the pair the pass was started for. Named + // the way the diff names the two sides: `exists_in_fork` is this workspace, `exists_in_source` + // the one it is compared against. + const forkWs = currentWorkspaceId + const sourceWs = parentWorkspaceId for (const diff of itemsToFetch) { + if (!alive) return const key = getItemKey(diff) - // Skip if already cached or loading - if (summaryCache[key]) continue + // Asked for already — including one still in flight, and one whose answer was nothing. + if (summariesRequested.has(key)) continue + summariesRequested.add(key) - // Mark as loading summaryCache[key] = { loading: true } - // Fetch from both workspaces in parallel + // Only the sides that have the item. An item that is only ahead does not exist in the + // other workspace, so asking it there is a 404 by construction — a guaranteed-failing + // request per one-sided row, and a console error for each. const [currentSummary, parentSummary] = await Promise.all([ - fetchSummary(diff.kind, diff.path, currentWorkspaceId), - fetchSummary(diff.kind, diff.path, parentWorkspaceId) + diff.exists_in_fork ? fetchSummary(diff.kind, diff.path, forkWs) : undefined, + diff.exists_in_source ? fetchSummary(diff.kind, diff.path, sourceWs) : undefined ]) + if (!alive) return summaryCache[key] = { current: currentSummary, @@ -561,18 +620,32 @@ (d) => ['flow', 'script', 'app', 'raw_app'].includes(d.kind) || isTriggerOrScheduleKind(d.kind) ) + const pair = [ + { workspace: currentWorkspaceId, side: 'fork' as const }, + { workspace: parentWorkspaceId, side: 'source' as const } + ] for (const diff of itemsWithOnBehalfOf) { - for (const workspace of [currentWorkspaceId, parentWorkspaceId]) { + for (const { workspace, side } of pair) { + if (!alive) return const workspacedKey = getWorkspacedKey(workspace, getItemKey(diff)) - if (onBehalfOfInfo[workspacedKey] !== undefined) continue + // Marked before the request, not after it answers: a guard keyed on the value it + // stores (`undefined` either way) would ask again on every pass, forever. + if (onBehalfOfRequested.has(workspacedKey)) continue + onBehalfOfRequested.add(workspacedKey) + // The side that does not have the item has nothing to say about who it runs as, and + // asking produces a 404 rather than an answer. + if (!(side === 'fork' ? diff.exists_in_fork : diff.exists_in_source)) continue try { - onBehalfOfInfo[workspacedKey] = await getOnBehalfOf( - diff.kind as Kind, - diff.path, - workspace - ) + // The same read the summary loop does, for the kinds that answer both from one + // object; the rest keep their own endpoint. + const email = SHARED_READ_KINDS.includes(diff.kind) + ? (await readItem(diff.kind, diff.path, workspace)).onBehalfOf + : await getOnBehalfOf(diff.kind as Kind, diff.path, workspace) + if (!alive) return + onBehalfOfInfo[workspacedKey] = email } catch { + if (!alive) return onBehalfOfInfo[workspacedKey] = undefined } } @@ -1004,11 +1077,16 @@ ) ) - // Fetch summaries and on_behalf_of_email when comparison data loads + // Fetch summaries and on_behalf_of_email when comparison data loads. `untrack`: both fetchers + // write the state they also read, and without this the effect depends on their writes and + // re-runs on each one. $effect(() => { - if (comparison?.diffs) { - fetchSummaries(comparison.diffs) - fetchOnBehalfOfInfo(comparison.diffs) + const diffs = comparison?.diffs + if (diffs) { + untrack(() => { + fetchSummaries(diffs) + fetchOnBehalfOfInfo(diffs) + }) } }) @@ -1229,6 +1307,7 @@ onSelectAll={selectAll} onDeselectAll={deselectAll} emptyMessage={emptyDeployMessage} + loading={comparisonLoading} > {#snippet header()}
@@ -1650,13 +1729,13 @@ {#if isConflict || existsInBothWorkspaces} {#if diff.ahead > 0} - + {diff.ahead} ahead {/if} {#if diff.behind > 0} - + {diff.behind} behind {/if} diff --git a/frontend/src/lib/components/ContextBridge.svelte b/frontend/src/lib/components/ContextBridge.svelte new file mode 100644 index 0000000000..069bec0f42 --- /dev/null +++ b/frontend/src/lib/components/ContextBridge.svelte @@ -0,0 +1,18 @@ + + + +{@render children()} diff --git a/frontend/src/lib/components/CronBuilder.svelte b/frontend/src/lib/components/CronBuilder.svelte index bcf8508c12..3bceabb417 100644 --- a/frontend/src/lib/components/CronBuilder.svelte +++ b/frontend/src/lib/components/CronBuilder.svelte @@ -15,7 +15,7 @@ contentClasses="p-4" > {#snippet trigger()} - {/snippet} diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 5204a876b9..837569c575 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -375,7 +375,8 @@ } if (!isInstanceDatatable(datatable)) return { - reason: 'Only data tables stored in the instance database have roles and grants.', + reason: + "Roles and grants are not supported on data tables backed by a Postgres resource. Store the data table on Windmill's database or on an external Postgres instance (set up by a superadmin in instance settings) to use them.", ee: false } if (needsAdmin && !canManageDatatable) @@ -458,6 +459,11 @@ // default to open — actually be folded; a plain "expanded" set could never // close them, since the default would keep winning. // What the permissions drawer is open on, if anything: a schema, or a table. + let aclEditor = $state(undefined) + /** The open access editor lists the data table's roles, which the roles drawer can change. */ + export function refreshAcl() { + aclEditor?.refresh() + } let aclDrawer = $state<{ datatable: string | undefined; target: AclTarget } | undefined>( undefined ) @@ -1466,7 +1472,7 @@ (aclDrawer = undefined)}> (aclDrawer = undefined)} tooltip="Who owns this, and what each role may do with it." > @@ -1474,7 +1480,18 @@ {@const dt = aclDrawer.datatable ?? currentDatatable} {#if dt} {#key `${dt}~${JSON.stringify(aclDrawer.target)}`} - + onDatatableAction?.(dt, 'roles'), + blocker: accessBlocker(dt, true)?.reason + } + : undefined} + /> {/key} {/if} {/if} diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index bd016544b0..30fc93377d 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -311,6 +311,7 @@ function refreshRoles() { datatables.refetch() usableRoles.refetch() + dbManagerContent?.dbManager()?.refreshAcl() } async function handleExportSchema(explicitSource?: string) { diff --git a/frontend/src/lib/components/DBTableEditor.svelte b/frontend/src/lib/components/DBTableEditor.svelte index 4eb07aea5f..21cca65f63 100644 --- a/frontend/src/lib/components/DBTableEditor.svelte +++ b/frontend/src/lib/components/DBTableEditor.svelte @@ -105,7 +105,7 @@ const defaultColumnType = ( { postgresql: 'BIGSERIAL', - snowflake: 'varchar', + snowflake: 'VARCHAR', ms_sql_server: 'varchar', bigquery: 'string', mysql: 'varchar', diff --git a/frontend/src/lib/components/DatatablePicker.svelte b/frontend/src/lib/components/DatatablePicker.svelte index 53617acf36..17ad6f3361 100644 --- a/frontend/src/lib/components/DatatablePicker.svelte +++ b/frontend/src/lib/components/DatatablePicker.svelte @@ -17,6 +17,9 @@ selectInputClass?: string class?: string onClear?: () => void + /** Size the input to its content, like Path's folder picker. */ + useContentEditable?: boolean + RightIcon?: any } let { @@ -27,7 +30,9 @@ placeholder = undefined, selectInputClass = '', class: className = '', - onClear = undefined + onClear = undefined, + useContentEditable = false, + RightIcon = undefined }: Props = $props() let datatables = usePromise(() => @@ -47,6 +52,8 @@ {placeholder} inputClass={selectInputClass} {onClear} + {useContentEditable} + {RightIcon} /> {#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })} diff --git a/frontend/src/lib/components/DevWorkspaceBadge.svelte b/frontend/src/lib/components/DevWorkspaceBadge.svelte new file mode 100644 index 0000000000..e86e4593c8 --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceBadge.svelte @@ -0,0 +1,56 @@ + + + +{devBadgeText(label)} diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte index e148d90c22..ff9932fa87 100644 --- a/frontend/src/lib/components/DevWorkspaceSetting.svelte +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -1,7 +1,8 @@ -
+
{#if !hidePath}
@@ -162,9 +111,9 @@ {#if segments} {#each segments.dirs as dir, i (dir.fullPath)} {@const dKey = dirKey('all', dir.fullPath)} + {/each} + {/if} + {#if segments?.leaf} {@const leafKey = leafKeyFor(kind, segments.leaf.fullPath)} {@const leafParent = segments.dirs[segments.dirs.length - 1]?.fullPath} + - + {#if pathEditable} - pathPopoverOpen, setPathPopoverOpen} - > - {#snippet trigger()} -
{/if} diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index b4bc4471a2..0a65701b06 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -303,7 +303,12 @@ {#if showBanner} - -
-
- -
-
- -
- - {isDevWorkspace - ? `${devLabelWord(currentWorkspaceData?.dev_workspace_label)} workspace of` - : 'Fork of'} - {parentWorkspaceData?.name}{#if parentWorkspaceData?.name !== parentWorkspaceId} - ({parentWorkspaceId}){/if} - -
- - {#if loading} - Checking for changes... - {:else if error} - - {error} - - {:else if comparison} -
- {#if comparison.summary.total_diffs > 0} - - {forkAheadBehindMessage(changesAhead, changesBehind)} - {parentWorkspaceId} over {comparison - .summary.total_diffs} items - - - - - {#if ciTestTotal > 0} - - - {#if ciTestFailing > 0} -
- - CI: {ciTestFailing} failing -
- {:else if ciTestRunning > 0} -
- - CI: {ciTestRunning} running -
- {:else} -
- - CI: {ciTestPassing} passing -
- {/if} - {/if} - - {#if comparison.summary.conflicts > 0} - - -
- - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 - ? 's' - : ''} -
- {/if} - {:else if comparison.skipped_comparison} - - This {currentNoun} was created before the addition of certain windmill features, and - therefore the changes with its parent workspace cannot be displayed. - {:else if showDraftsOnly} - - This workspace has {draftCount} draft{draftCount !== 1 ? 's' : ''} - - {:else} - Everything is up to date - {/if} -
- {/if} -
- -
- -
-
-
-
+ + {/if} + +{#snippet forkAction()} + {@const total = comparison?.summary.total_diffs ?? 0} + {@const conflicts = comparison?.summary.conflicts ?? 0} + + {#if loading} + Checking for changes… + {:else if error} + Comparison failed + {:else if showDraftsOnly} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + {:else if total > 0} + + + {#if changesAhead > 0} + openCompareDir('deploy_to')} + title="{changesAhead} ahead of {parentWorkspaceId} — review what this {currentNoun} has to give" + > + + {changesAhead}{compactCounts ? '' : ' ahead'} + + {/if} + {#if changesBehind > 0} + openCompareDir('update')} + title="{changesBehind} behind {parentWorkspaceId} — review what this {currentNoun} has yet to take" + > + + {changesBehind}{compactCounts ? '' : ' behind'} + + {/if} + {#if conflicts > 0} + + openCompareDir('deploy_to')} + title="{conflicts} conflicting item{conflicts !== 1 ? 's' : ''} — review them" + > + + {conflicts}{compactCounts ? '' : ` conflict${conflicts !== 1 ? 's' : ''}`} + + {/if} + + {#if draftCount > 0} + + + {draftCount}{compactCounts ? '' : ` draft${draftCount !== 1 ? 's' : ''}`} + + {/if} + {/if} + {#if showDraftsOnly} + + {:else} + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/NavBreadcrumb.svelte b/frontend/src/lib/components/NavBreadcrumb.svelte new file mode 100644 index 0000000000..3081bcca4c --- /dev/null +++ b/frontend/src/lib/components/NavBreadcrumb.svelte @@ -0,0 +1,434 @@ + + + +{#snippet workspaceDisc()} + + + + + + + +{/snippet} + +{#snippet itemKindIcon()} + + {#if item?.kind}{/if} +{/snippet} + +{#snippet sectionKindIcon()} + {#if section?.kind}{/if} +{/snippet} + +{#snippet folderIcon()} + +{/snippet} + +{#snippet userIcon()} + +{/snippet} + +{#snippet envBadgeMark()} + {#if envBadge === 'dev'} + + {:else if envBadge === 'prod'} + prod + {/if} +{/snippet} + +{#snippet slash(extra = '')} + + +{/snippet} + + diff --git a/frontend/src/lib/components/NoDirectDeployAlert.svelte b/frontend/src/lib/components/NoDirectDeployAlert.svelte index d1017f7347..a3a249caf5 100644 --- a/frontend/src/lib/components/NoDirectDeployAlert.svelte +++ b/frontend/src/lib/components/NoDirectDeployAlert.svelte @@ -16,7 +16,8 @@ import { findCanonicalDevWorkspace } from '$lib/utils/workspaceHierarchy' import { devLabelKey, devLabelNoun } from '$lib/utils/devWorkspaceLabel' import { switchWorkspace } from '$lib/storeUtils' - import { Badge, Button } from './common' + import { Button, ButtonType } from './common' + import { twMerge } from 'tailwind-merge' import Popover from './meltComponents/Popover.svelte' import Toggle from './Toggle.svelte' import { GitFork, Lock, ShieldOff } from 'lucide-svelte' @@ -83,28 +84,35 @@ ? `Edits in ${devLabelKey(canonicalDev.dev_workspace_label)}` : 'Edits restricted' ) + + // The Popover renders its trigger as a + - {#if customUi?.topBar?.path != false} -
- onNavigate?.(item)} - /> -
+ {#if ownsPageHeader} + + + + {:else} +
+
+
+ {@render languageButton(condensedHeader ? 18 : 24)} + {#if customUi?.topBar?.path != false} +
+ onNavigate?.(item)} + /> +
+ {/if} + {@render autosaveIndicator()} + {#if !condensedHeader} + {@render settingsBadges()} + {/if} +
+ + +
+ + {#if $enterpriseLicense && initialPath != '' && !inSessionPane} + {/if} - {#if opWorkspace} - - {/if} - {#if !condensedHeader} - {@const canOpenRuntime = - customUi?.topBar?.settings != false && - customUi?.settingsPanel?.disableRuntime !== true} - { - selectedTab = 'runtime' - metadataOpen = true - } - : undefined} + + +
+ + {@render rightButtons()} +
+
+ {/if} + + {#snippet scriptSummary()} + + {#if script} + +
+ {#snippet summaryText()} + {emptyString(script.summary) ? 'Add a summary...' : script.summary} + {/snippet} + {#if customUi?.topBar?.editablePath != false || customUi?.topBar?.editableSummary != false} + + {:else} + {@render summaryText()} {/if}
+ {/if} + {/snippet} - -
+ + {#snippet scriptPathTrigger(pathLabel: Snippet, triggerClass: string)} + + {#if script} + + {:else} + {@render pathLabel()} + {/if} + {/snippet} + {#snippet scriptMarks()} + {#if script} + + {@render settingsBadges('pl-1.5')} + {/if} + {/snippet} + + {#snippet scriptHeaderActions()} + {#if script} + + {@render autosaveIndicator()} {#if $enterpriseLicense && initialPath != '' && !inSessionPane} {/if} + {@render rightButtons()} + {/if} + {/snippet} - -
+ {#snippet languageButton(size: number)} + + {/snippet} - {#snippet settingsButton()} - {#if customUi?.topBar?.settings != false} - - {/if} - {/snippet} - {#snippet diffButton()} - {#if customUi?.topBar?.diff != false} - {@const isDraftOnly = savedScript?.no_deployed === true} - {@const diffDisabled = !savedScript || !diffDrawer || isDraftOnly} - {@const diffTitle = isDraftOnly - ? 'Deploy this script once to compare against the deployed version' - : 'Diff'} - -
- -
- {/if} - {/snippet} - {#if compactTopbar} - - {#snippet buttonReplacement()} - + + {/if} + {/snippet} + {#if compactTopbar} + + {#snippet buttonReplacement()} + - - - + + {/if} + +{#snippet draftsAction()} + + {#if phone} + + {:else} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/WorkspaceItemKindIcon.svelte b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte new file mode 100644 index 0000000000..9d07f6ef89 --- /dev/null +++ b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte @@ -0,0 +1,19 @@ + + + +{#if kind === 'flow'} + +{:else if kind === 'app'} + +{:else} + +{/if} diff --git a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte index 77fe52b421..37a27f3812 100644 --- a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte +++ b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte @@ -2,8 +2,8 @@ import { userWorkspaces, type UserWorkspace } from '$lib/stores' import { findWorkspaceRoot } from '$lib/utils/workspaceHierarchy' import { forkAccentStyle } from '$lib/utils/forkColor' - import { Badge, Button } from '$lib/components/common' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import { Button } from '$lib/components/common' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { Building, ChevronDown, EllipsisVertical, GitFork } from 'lucide-svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { type Item } from '$lib/utils' @@ -171,12 +171,12 @@ {#if pendingFork} (new) {:else if currentWs?.is_dev_workspace} - {devBadgeText(currentWs.dev_workspace_label)} + {/if} {/if} diff --git a/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte b/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte index 0483bbd272..767240a60f 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte @@ -30,6 +30,42 @@ nullable: 'YES' | 'NO' } + // Snowflake's information_schema reports canonical names (NUMBER, TEXT, + // TIMESTAMP_NTZ, ...), which the pinned parser wasm only knows by a few aliases. + function snowflakeParserAlias(typ: string): string { + switch (typ) { + case 'number': + case 'decimal': + case 'numeric': + case 'float4': + case 'float8': + case 'double': + case 'double precision': + case 'real': + return 'float' + case 'integer': + case 'bigint': + case 'smallint': + case 'tinyint': + case 'byteint': + return 'int' + case 'text': + case 'string': + case 'char': + case 'character': + return 'varchar' + case 'varbinary': + return 'binary' + case 'datetime': + case 'timestamp_ntz': + case 'timestamp_ltz': + case 'timestamp_tz': + return 'timestamp' + default: + return typ + } + } + function parseSQLArgs(field: string, dbType: DbType): string { let rawType = '' switch (dbType) { @@ -43,7 +79,7 @@ rawType = parse_bigquery(field) break case 'snowflake': - rawType = parse_snowflake(field) + rawType = parse_snowflake(snowflakeParserAlias(field.toLowerCase())) break case 'ms_sql_server': rawType = parse_mssql(field) @@ -97,12 +133,27 @@ schemaProperty.default = field.defaultValue } } - if (field.type === 'timestamp without time zone' || field.type === 'timestamp') { + const fieldType = dbType === 'snowflake' ? field.type.toLowerCase() : field.type + if ( + fieldType === 'timestamp without time zone' || + fieldType === 'timestamp' || + fieldType === 'timestamp_ntz' + ) { schemaProperty.format = 'naive-date-time' } - if (field.type === 'timestamp with time zone' || field.type === 'timestamptz') { + if ( + fieldType === 'timestamp with time zone' || + fieldType === 'timestamptz' || + fieldType === 'timestamp_tz' || + fieldType === 'timestamp_ltz' + ) { schemaProperty.format = 'date-time' } + if (dbType === 'snowflake' && fieldType === 'date') { + schemaProperty.format = 'date' + // DateInput defaults to dd-MM-yyyy, which Snowflake's AUTO input format rejects + schemaProperty.dateFormat = 'yyyy-MM-dd' + } properties[field.name] = schemaProperty diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 073d6c3e7d..ac0d2b1cb6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -86,7 +86,8 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: AppEditorProps = $props() migrateApp(untrack(() => app)) @@ -885,6 +886,7 @@ {#if !$userStore?.operator} {#if $appStore} hideBottomPanel()} /> {#if $mode === 'preview'} - +
{/if} - +
void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } let { @@ -142,10 +147,11 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: Props = $props() - /** Mirror of the path the user is editing in the pen popover. Initialized + /** Mirror of the path the user is editing in the rename popover. Initialized * once from `newPath` (or a synthesized path for new apps) and only * updated by user input from then on — we deliberately do NOT sync from * `newPath` afterwards so the user's in-flight rename isn't clobbered by @@ -223,6 +229,18 @@ // the preview to the deployed version. const inSessionPane = !!getContext('aiChatManager') + // The header's buttons render under the page header, not under this component, so the contexts + // they look up have to travel with them. The app's own two carry everything the canvas toggles, + // the panel buttons, Debug runs and the preview switch read. + /** Held by the rename popover while it is open; the band's trail reads it so the path does not + * reflow under the pointer as the user types. */ + let pathSnapshot = $state(undefined) + + const headerContexts = new Map([ + ['AppViewerContext', getContext('AppViewerContext')], + ['AppEditorContext', getContext('AppEditorContext')] + ]) + const loading = $state({ publish: false, save: false @@ -243,7 +261,24 @@ // Top-bar responsive collapse — container width, not viewport. let topbarWidth = $state(0) - const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720) + // In the page header the buttons share the row with the breadcrumb, so they shed the canvas + // toggles and fold Debug runs into the menu sooner: the whole group is ~840px and the trail + // with the summary takes ~430px of the same row. + // Measured on the band rather than estimated: with the canvas toggles shown the actions alone + // are 826-911px and the trail with the summary another ~500, so the row overflows the bar at + // every width up to about 1470 — 1350 was not high enough either. Below this the toggles stand + // down and the menu carries all three, which is what keeps them reachable rather than merely + // out of the way. + const compactBelow = $derived(ownsPageHeader ? 1500 : 720) + const compactTopbar = $derived.by(() => { + const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth + return w > 0 && w < compactBelow + }) + // A phone's bar holds the trail and Deploy. Which panels are open, the export and the + // editor/preview switch join the menu compact already shows. + const phoneTopbar = $derived( + ownsPageHeader && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR + ) function closeSaveDrawer() { saveDrawerOpen = false @@ -590,7 +625,49 @@ selectedJobId = $jobs[$jobs.length - 1] } $jobsDrawerOpen = true - }, + } + }, + // The canvas toggles stand down with the bar, and the breakpoint among them is the + // only way onto the mobile layout — so each one keeps an entry here. + { + displayName: $breakpoint === 'sm' ? 'Computer view' : 'Mobile view', + icon: $breakpoint === 'sm' ? Laptop2 : Smartphone, + action: () => ($breakpoint = $breakpoint === 'sm' ? 'lg' : 'sm') + }, + // The setting the mobile view is edited for, which otherwise lives beside the + // toggle group that stood down: without it here a deployed app could be put on + // the mobile layout at this width and never taken off it. + ...($breakpoint === 'sm' + ? [ + { + displayName: $app?.mobileViewOnSmallerScreens + ? 'Mobile view on small screens: on' + : 'Mobile view on small screens: off', + icon: Smartphone, + action: () => { + if ($app) $app.mobileViewOnSmallerScreens = !$app.mobileViewOnSmallerScreens + } + } + ] + : []), + { + displayName: $app?.fullscreen ? 'Centered canvas' : 'Full-width canvas', + icon: $app?.fullscreen ? AlignHorizontalSpaceAround : Expand, + action: () => { + if ($app) $app.fullscreen = !$app.fullscreen + } + }, + { + displayName: + $app?.darkMode === undefined + ? 'Theme: automatic' + : $app.darkMode + ? 'Theme: dark' + : 'Theme: light', + icon: $app?.darkMode === undefined ? SunMoon : $app.darkMode ? Moon : Sun, + // Cycles auto → light → dark, the order the toggle group reads in. + action: () => + setTheme($app?.darkMode === undefined ? false : $app.darkMode ? undefined : true), separatorBottom: true } ] @@ -913,141 +990,232 @@ -
- + ` placeholder, and the trail would name that instead of the path Deploy + // will create. Same fallback chain the rename sites read. + // Frozen while the rename popover is open so the trail holds still as the user types. + kind: 'app', + path: pathSnapshot ?? (newEditedPath || $appPath || newPath || undefined), + summaryContent: appSummary, + pathTrigger: appPathTrigger + }} + actions={appHeaderActions} + contexts={headerContexts} + separator="always" + /> +{:else} +
+ -
-
- (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} - /> -
-
- {#if $app} - { - $app.fullscreen = detail === 'true' - }} - > - {#snippet children({ item })} - - - {/snippet} - - {/if} - {#if $app} - { - const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined - setTheme(theme) - }} - selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} - > - {#snippet children({ item })} - - - - {/snippet} - - {/if} -
- - {#snippet children({ item })} - - - {#if $breakpoint === 'sm'} - - {/if} - {/snippet} - -
-
- {#if $workspaceStore} -
- +
+ (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} />
- {/if} -
+ {@render canvasToggles()} + {@render autosaveIndicator()} +
+ {@render panelButtons()} + {@render awarenessMark()} + {@render appActions()} +
+{/if} + + +{#snippet appPathTrigger(pathLabel: Snippet, triggerClass: string)} + +{/snippet} + +{#snippet appSummary()} + +
+ {#snippet summaryText()} + {emptyString($summary) ? 'Add a summary...' : $summary} + {/snippet} + +
+{/snippet} + +{#snippet appHeaderActions()} + +
+ {@render canvasToggles()} + {@render autosaveIndicator()} + {#if !phoneTopbar} + {@render panelButtons()} + {@render awarenessMark()} + {/if} + {@render appActions()} +
+{/snippet} + +{#snippet canvasToggles()} +
+ {#if $app} + { + $app.fullscreen = detail === 'true' + }} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} + {#if $app} + { + const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined + setTheme(theme) + }} + selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} + > + {#snippet children({ item })} + + + + {/snippet} + + {/if} +
+ + {#snippet children({ item })} + + + {#if $breakpoint === 'sm'} + + {/if} + {/snippet} + +
+
+{/snippet} + +{#snippet autosaveIndicator()} + {#if $workspaceStore} +
+ +
+ {/if} +{/snippet} + +{#snippet panelButtons()} {#if $mode !== 'preview'}
{/if} +{/snippet} + +{#snippet awarenessMark()} {#if $enterpriseLicense && $appPath != '' && !inSessionPane}
{/if} +{/snippet} + +{#snippet appActions()}
@@ -1131,14 +1305,19 @@
+ - +
-
+{/snippet} diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index e004c17a04..cb56b37498 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -13,13 +13,18 @@ import { sendUserToast } from '$lib/toast' import PublicApp from '$lib/components/apps/editor/PublicApp.svelte' import PublicAppFrame from '$lib/components/apps/editor/PublicAppFrame.svelte' + import PageHeaderContent from '$lib/components/PageHeaderContent.svelte' + import DropdownV2 from '$lib/components/DropdownV2.svelte' + import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte' + import type { Item } from '$lib/utils' import { Button } from '$lib/components/common' import { AppService, OpenAPI } from '$lib/gen' - import type { UserExt } from '$lib/stores' + import { userStore, type UserExt } from '$lib/stores' import { canWrite } from '$lib/utils' import { getUserExt } from '$lib/user' - import { Pen } from 'lucide-svelte' + import { ExternalLink, Pen } from 'lucide-svelte' import { page } from '$app/state' + import { isMenuHidden } from '$lib/components/sessions/sessionMode.svelte' import { setOperatingWorkspace, useOperatingUser @@ -30,10 +35,15 @@ path, onEdit, onLoadState, - syncHashToUrl = true + syncHashToUrl = true, + ownsPageHeader = false }: { workspace: string path: string + /** True for the route's own page: the app becomes what the page header names. A host that + * renders this inside something else — a session's preview panel, whose band names the + * session — leaves it false, and the app keeps its Edit over its own canvas. */ + ownsPageHeader?: boolean /** Handle Edit in place instead of following `editHref`. An AI session shows * this viewer inside a preview tab, where a plain link would navigate the whole * page out of the session rather than flipping the tab to its editor. */ @@ -70,6 +80,12 @@ * app point at /apps/get all over the app), so only the app can say which * editor the Edit button must open. */ let isRawApp = $state(false) + /** The app's execution mode, from the same lite read as the permissions and for the same + * reason: a sandboxed low-code app loads inside the opaque iframe, so `app` here stays + * undefined and cannot say whether the app answers at a public url. `policy.execution_mode` is + * where the lite payload carries it; the flat field beside it is the apps *list* row's + * spelling (ListableApp, which is what AppRow reads). */ + let executionMode = $state(undefined) let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined @@ -85,6 +101,11 @@ const hideEditBtn = page.url.searchParams.get('hideEditBtn') === 'true' const hideRefreshBar = page.url.searchParams.get('hideRefreshBar') === 'true' + const showEdit = $derived(canWriteApp && !hideEditBtn) + // Decides where Edit goes, not whether it appears: without the workspace navigation there is + // no band to put it in. Same rule the layout hides the sidebar by. + const menuHidden = $derived(isMenuHidden(page.url)) + // Embedder side: mint a scoped embed token (by path) from the member's session. async function fetchEmbedToken(opts?: { sdkConsent?: boolean }): Promise<{ token?: string }> { const headers: Record = {} @@ -155,48 +176,142 @@ const lite: any = await AppService.getAppLiteByPath({ workspace, path }) appPerms = { path: lite?.path ?? path, extraPerms: lite?.extra_perms ?? {} } isRawApp = !!lite?.raw_app + executionMode = lite?.policy?.execution_mode ?? lite?.execution_mode } catch (_) { appPerms = undefined } } + /** Whether the app answers at a public url at all. Read from the app this viewer has already + * loaded, the way the apps table decides the same thing (AppRow) — the secret itself is a + * request, and asking for one on every app view to decide whether to draw a button is a + * request per view that most views never use. */ + const hasPublicUrl = $derived(executionMode === 'anonymous') + /** Resolved when the button is used, not when it is drawn. */ + async function openPublicUrl() { + // The tab is opened inside the click itself and filled once the secret arrives. Safari only + // lets a gesture open the tab it opens synchronously, so opening it after the await is + // blocked there — silently, since a blocked popup reports nothing. + const tab = window.open('about:blank', '_blank') + try { + const secret = await AppService.getPublicSecretOfApp({ workspace, path }) + if (!secret) { + tab?.close() + sendUserToast('This app has no public url', true) + return + } + // Built from this viewer's workspace rather than the navigation one, like everything + // else here: the two differ inside a session's preview panel. + const url = `${window.location.origin}${base}/public/${workspace}/${secret}` + if (tab) tab.location.href = url + else sendUserToast('Allow pop-ups for this site to open the app’s public url', true) + } catch (e: any) { + tab?.close() + sendUserToast('Could not open the public url: ' + (e?.body ?? e?.message ?? e), true) + } + } + $effect(() => { if (workspace && path) loadPerms() }) + + // Both of this page's buttons carry a label, and a phone's bar has room for neither beside the + // app's path — so below that width they become one menu. Unmeasured (0) counts as wide: the bar + // measures itself on mount, and starting compact would pop them out a frame later. + const compact = $derived(pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR) + const compactItems: Item[] = $derived([ + ...(hasPublicUrl + ? [ + { + displayName: 'Public url', + icon: ExternalLink, + action: () => openPublicUrl() + } + ] + : []), + { + displayName: 'Edit', + icon: Pen, + href: onEdit ? undefined : editHref, + action: onEdit ? () => onEdit() : undefined + } + ]) - { - refresh = requestTokenRefresh - loadApp() - }} -> - {#snippet viewer()} - loadApp()} - > - {/snippet} - +
+ { + refresh = requestTokenRefresh + loadApp() + }} + > + {#snippet viewer()} + loadApp()} + > + {/snippet} + +
-{#if canWriteApp && !hideEditBtn} -
- + +{#if ownsPageHeader} + +{/if} + + +{#if showEdit && (menuHidden || !ownsPageHeader)} +
+ {@render editAction()}
{/if} + +{#snippet editAction()} + {#if compact} + + + {:else} + {#if hasPublicUrl} + + + {/if} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte index 7a72ac62b3..4e20301043 100644 --- a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte +++ b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte @@ -8,9 +8,11 @@ interface Props { loading?: boolean + /** Drops the two labels, for a bar with no room for them: the pen and the eye carry it. */ + iconOnly?: boolean } - let { loading = false }: Props = $props() + let { loading = false, iconOnly = false }: Props = $props() const { mode, jobs, jobsById } = getContext('AppViewerContext') @@ -24,7 +26,8 @@ > {#snippet children({ item })} import { applyDarkModeVariant } from '$lib/darkModeVariant' - import { enterpriseLicense, userStore, type UserExt } from '$lib/stores' - import { base } from '$app/paths' - import { page } from '$app/state' - import Login from '$lib/components/Login.svelte' + import type { UserExt } from '$lib/stores' import { isCloudHosted } from '$lib/cloud' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import WindmillIcon from '$lib/components/icons/WindmillIcon.svelte' - import { getContext, setContext } from 'svelte' - import { - EMBED_NAV_CONTEXT_KEY, - IS_APP_PUBLIC_CONTEXT_KEY, - type EditorBreakpoint, - type EmbedNav - } from '../types' - import { UserService, type AppWithLastVersion, type GlobalWhoamiResponse } from '$lib/gen' - import { urlParamsToObject } from '$lib/utils' - import { goto } from '$app/navigation' - import { loadAppPreview } from './loadAppPreview' + import { setContext } from 'svelte' + import { IS_APP_PUBLIC_CONTEXT_KEY } from '../types' + import type { AppWithLastVersion } from '$lib/gen/types.gen' import RawAppPreview from '$lib/components/raw_apps/RawAppPreview.svelte' import type { Runnable } from '$lib/components/raw_apps/rawAppPolicy' - import { twMerge } from 'tailwind-merge' - import { writable } from 'svelte/store' + import { loadAppCss } from './publicAppApi' let { notExists, @@ -33,6 +20,7 @@ app, workspace, user = undefined, + license = undefined, inWorkspace = false, hideRefreshBar = false, syncHashToUrl = true @@ -46,11 +34,13 @@ onLoginSuccess: () => void app: (AppWithLastVersion & { value: any; workspace_id?: string }) | undefined workspace: string | undefined - /** The viewer's membership in `workspace`, for the app's `ctx`. Without it `ctx` falls - * back to `userStore`, which describes the workspace the page is navigated to — the - * same one on every standalone viewer route, but not inside an AI session's preview - * tab, where the app belongs to the session's workspace. */ + /** The viewer's membership in `workspace`, for the app's `ctx`. Callers pass it + * explicitly — `$userStore` describes the workspace the page is navigated to, which + * is not the app's inside an AI session's preview tab. */ user?: UserExt | undefined + /** The instance license, which fades the "Powered by" badge and lets a low-code app's + * custom CSS apply. */ + license?: string | undefined /** * In-workspace rendering (`/apps/get`, `/app_embed`): keep exact parity * with the pre-sandbox member viewer — no "Powered by Windmill" badge, no @@ -65,23 +55,11 @@ // Use workspace from props or from app.workspace_id (for custom path responses) let effectiveWorkspace = $derived(workspace ?? app?.workspace_id) - // `ctx.username` / `ctx.groups` are per-workspace, and they sit next to `ctx.workspace` - // in the same object, so they have to come from the same workspace it names. - let ctxUser = $derived(user ?? $userStore) - // On the public surfaces (untrusted distribution) runnable-authored html/svg needs // the viewer's approval before it renders, unless the app sandbox isolates it. The // in-workspace viewer renders it verbatim. See getAppMarkupTrust. setContext(IS_APP_PUBLIC_CONTEXT_KEY, !inWorkspace) - // WIN-2006: inside the opaque viewer iframe, navigations to other routes - // (navbar "app" items) must happen on the TOP page — the iframe is cookieless, - // so navigating it would just show a login screen. PublicAppFrame provides the - // relay; outside the opaque viewer this is undefined and goto works directly. - const embedNav = getContext(EMBED_NAV_CONTEXT_KEY) - - const breakpoint = writable('lg') - const darkMode = window.localStorage.getItem('dark-mode') ?? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light') @@ -93,135 +71,75 @@ } // This route bypasses the (root) layout, so restore the variant class too. applyDarkModeVariant() - - let globalUser = $state(undefined) - async function loadGlobalUser() { - try { - globalUser = await UserService.globalWhoami() - } catch (error) { - console.error(error) - } - // const user = await fetch('/api/global/user') - // console.log(user) - } - - // Only the no-access page reads it, to tell a signed-in non-member which - // workspace the app belongs to. - $effect(() => { - if (noPermission && !guestAppPath && !$userStore && !globalUser) { - loadGlobalUser() - } - }) + {#if !inWorkspace} -
- Powered by   Windmill -
+ Powered by   Windmill +
+ {/await} {/if} -{#if notExists} -
There was an error loading the app, is the url correct? Go to Windmill -
-{:else if noPermission} - {#if guestAppPath && !$userStore} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
-
- {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} - You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of - {:else}You must be logged in and have read access to this app{/if}
- {/if} -
- {#if !jwtError} - - {/if} -
+{#if notExists || noPermission} + {#await Promise.all([loadAppCss(), import('./PublicAppAccess.svelte')]) then [, { default: PublicAppAccess }]} + + {/await} {:else if app} {#key app} {#if app.raw_app && effectiveWorkspace} - } - {syncHashToUrl} - /> + +
+ } + {syncHashToUrl} + /> +
{:else if app.raw_app && !effectiveWorkspace} -
- - Unable to load raw app: workspace information is missing. - -
+ {#await Promise.all([loadAppCss(), import('$lib/components/common/alert/Alert.svelte')]) then [, { default: Alert }]} +
+ + Unable to load raw app: workspace information is missing. + +
+ {/await} {:else} -
- {#await loadAppPreview()} - - {:then Module} - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> - {:catch} -
- Reload the page to try again. -
- {/await} -
+ {#await Promise.all([loadAppCss(), import('./PublicLowCodeApp.svelte')]) then [, { default: PublicLowCodeApp }]} + + {/await} {/if} {/key} {:else} - + {#await loadAppCss() then} + + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte new file mode 100644 index 0000000000..9a993b4972 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte @@ -0,0 +1,81 @@ + + +{#if notExists} +
There was an error loading the app, is the url correct? Go to Windmill +
+{:else} + {#if guestAppPath && !user} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+
+ {#if user}You are logged in but have no read access to this app{:else if globalUser && workspace} + You are logged in but are not a member of the workspace {workspace} this app is part of + {:else}You must be logged in and have read access to this app{/if}
+ {/if} +
+ {#if !jwtError} + { + dropPrefetched() + onLoginSuccess() + }} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> + {/if} +
+{/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index e994767488..dbb9b4b88b 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -23,19 +23,43 @@ * bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`. */ import { BROWSER } from 'esm-env' - import { OpenAPI, UserService } from '$lib/gen' + import { OpenAPI } from '$lib/gen/core/OpenAPI' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' - import Login from '$lib/components/Login.svelte' - import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' + import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils/queryParams' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' import { loadAppPreview } from './loadAppPreview' - import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' - import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' + import { dropPrefetched, loadAppCss } from './publicAppApi' + import { + hasStoredSdkConsent, + sdkConsentCovers, + storeSdkConsent + } from '$lib/components/raw_apps/sdkScopes' + + // Everything but a rendered app (sign-in, errors, the SDK consent prompt) loads on + // demand: a raw app that renders right away needs none of it. See publicAppApi.ts. + let frameUi: + | Promise<{ + Alert: typeof import('$lib/components/common/alert/Alert.svelte').default + Login: typeof import('$lib/components/Login.svelte').default + RawAppSdkConsent: typeof import('$lib/components/raw_apps/RawAppSdkConsent.svelte').default + }> + | undefined + function loadFrameUi() { + return (frameUi ??= Promise.all([ + loadAppCss(), + import('$lib/components/common/alert/Alert.svelte'), + import('$lib/components/Login.svelte'), + import('$lib/components/raw_apps/RawAppSdkConsent.svelte') + ]).then(([, alert, login, consent]) => ({ + Alert: alert.default, + Login: login.default, + RawAppSdkConsent: consent.default + }))) + } type EmbedToken = { token?: string | null @@ -231,7 +255,8 @@ if (offerSignIn && accountSession === 'unknown') { // Workspace-less, so it answers for an account and never for a guest // (pinned to its workspace) or for nobody. - UserService.getCurrentEmail() + import('$lib/gen') + .then(({ UserService }) => UserService.getCurrentEmail()) .then(() => (accountSession = 'held')) .catch(() => (accountSession = 'none')) } @@ -245,8 +270,12 @@ let staleGuestLogoutFailed = $state(false) $effect(() => { if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) { - UserService.logout() - .then(() => (staleGuestCleared = true)) + import('$lib/gen') + .then(({ UserService }) => UserService.logout()) + .then(() => { + dropPrefetched() + staleGuestCleared = true + }) .catch(() => (staleGuestLogoutFailed = true)) } }) @@ -388,7 +417,7 @@ appPath = resp.app_path ?? appPath workspaceId = resp.workspace_id ?? workspaceId const granted = resp.sdk_scopes ?? [] - if (!granted.every((s) => approved.includes(s))) { + if (!sdkConsentCovers(approved, granted)) { // The app was redeployed with more scopes between the prompt and the // mint. Never inject a token carrying scopes the viewer wasn't shown: // drop it and ask again with the new set. @@ -423,7 +452,10 @@ // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed // raw app. A low-code app's runtime downloads alongside the app payload. - if (!isRaw) loadAppPreview().catch(() => {}) + if (!isRaw) { + loadAppCss().catch(() => {}) + loadAppPreview().catch(() => {}) + } onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -533,6 +565,7 @@ onMount(() => { if (isViewer) { // Only a sandboxed low-code app is ever framed as a viewer. + loadAppCss().catch(() => {}) loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() @@ -565,105 +598,120 @@ {#if viewerReady} {@render viewer()} {:else if viewerOrphaned} -
- - This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your own - page, use the app's public URL without the wm_embed parameter. - -
+ {#await loadFrameUi() then { Alert }} +
+ + This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your + own page, use the app's public URL without the wm_embed parameter. + +
+ {/await} {:else} - + {#await loadAppCss() then} + + {/await} {/if} {:else if status === 'loading'} - -{:else if status === 'notExists' && !offerSignIn} -
- - There was an error loading the app, is the url correct? - Go to Windmill - -
-{:else if status === 'sdkPrompt'} - - -{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} - -{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} -
- - The app could not be reached to find out who may open it. Reload to try again. - -
-{:else if offerSignIn} - - {#if signInDidNotHelp} - -
- You are signed in, but this app is not open to you -
-
- It is open to the people it was shared with{guestAppPath - ? ', and to guests who have no Windmill account' - : ''}. Ask the person who shared it to give your account access. -
+ {#await loadAppCss() then} + + {/await} +{:else if status === 'ready'} + {#if unsandboxed} + + {@render viewer()} + {:else if isRaw} + + {@render viewer()} {:else} - {#if guestAppPath} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
- {/if} - {#if signInError} -
- {signInError} -
- {/if} -
- { - signInError = undefined - accountSession = 'unknown' - initEmbedder() - }} - onLoginError={(message) => (signInError = message)} - popup - guestApp={guestAppPath} - rd={page.url.pathname + page.url.search + page.url.hash} - /> -
+ + {/if} -{:else if unsandboxed} - - {@render viewer()} -{:else if isRaw} - - {@render viewer()} {:else} - - + {#await loadFrameUi() then { Alert, Login, RawAppSdkConsent }} + {#if status === 'notExists' && !offerSignIn} +
+ + There was an error loading the app, is the url correct? + Go to Windmill + +
+ {:else if status === 'sdkPrompt'} + + + {:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} + + {:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} +
+ + The app could not be reached to find out who may open it. Reload to try again. + +
+ {:else if offerSignIn} + + {#if signInDidNotHelp} + +
+ You are signed in, but this app is not open to you +
+
+ It is open to the people it was shared with{guestAppPath + ? ', and to guests who have no Windmill account' + : ''}. Ask the person who shared it to give your account access. +
+ {:else} + {#if guestAppPath} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+ {/if} + {#if signInError} +
+ {signInError} +
+ {/if} +
+ { + dropPrefetched() + signInError = undefined + accountSession = 'unknown' + initEmbedder() + }} + onLoginError={(message) => (signInError = message)} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> +
+ {/if} + {/if} + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte new file mode 100644 index 0000000000..f74be67166 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte @@ -0,0 +1,91 @@ + + +
+ {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
+ Reload the page to try again. +
+ {/await} +
diff --git a/frontend/src/lib/components/apps/editor/publicAppApi.ts b/frontend/src/lib/components/apps/editor/publicAppApi.ts new file mode 100644 index 0000000000..674cb0b622 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/publicAppApi.ts @@ -0,0 +1,82 @@ +// The public app routes (`/public/...`, `/a/...`) render a raw app as little more than its +// bundle's iframe, so they must not statically import the generated client, `$lib/stores` +// or `$lib/utils`: each drags in most of the app shell. `vite.config.js` +// (`assertLeanPublicAppRoutes`) fails the build if they do. Anything heavier is imported +// lazily by the branch that needs it. +import { OpenAPI } from '$lib/gen/core/OpenAPI' +import type { AppWithLastVersion, User } from '$lib/gen/types.gen' +import type { UserExt } from '$lib/stores' +import { mapUserToUserExt } from '$lib/userExt' + +export type PublicAppValue = AppWithLastVersion & { value: any; workspace_id?: string } + +declare global { + interface Window { + /** Responses `app.html` requested before any module loaded, keyed by URL. */ + __wmPublicAppPrefetch?: Record> + } +} + +/** Carries `status` like the generated client's `ApiError`, which callers branch on. */ +export class PublicApiError extends Error { + constructor(public status: number) { + super(`Request failed with status ${status}`) + } +} + +/** Prefetched responses answer for the credential the page loaded with; call this when it + * changes (a sign-in or logout on the page) so later reads go to the network. */ +export function dropPrefetched(): void { + delete window.__wmPublicAppPrefetch +} + +function takePrefetched(url: string): Promise | undefined { + const prefetched = window.__wmPublicAppPrefetch + const res = prefetched?.[url] + if (res) delete prefetched![url] + return res +} + +/** GET an API path with the page credential: the bearer token in `OpenAPI.TOKEN` when the + * share link carries one, the session cookie otherwise. A prefetched response is only + * taken in the cookie case, which is the only credential `app.html` sends. */ +export async function publicGet(path: string): Promise { + const url = `${OpenAPI.BASE}${path}` + const token = typeof OpenAPI.TOKEN === 'string' && OpenAPI.TOKEN ? OpenAPI.TOKEN : undefined + const res = await ((!token && takePrefetched(url)) || + fetch(url, { + // As the generated client does: the opaque viewer frame is cross-origin to the + // API, whose `Access-Control-Allow-Origin: *` refuses a credentialed request. + credentials: OpenAPI.WITH_CREDENTIALS ? OpenAPI.CREDENTIALS : undefined, + headers: token ? { Authorization: `Bearer ${token}` } : {} + })) + if (!res.ok) throw new PublicApiError(res.status) + return (res.headers.get('content-type')?.includes('json') ? res.json() : res.text()) as T +} + +export async function getUserExtOrUndefined(workspace: string): Promise { + try { + return mapUserToUserExt( + await publicGet(`/w/${encodeURI(workspace)}/users/whoami`), + workspace + ) + } catch { + return undefined + } +} + +/** Only fades the "Powered by" badge, so a failure is not worth surfacing. */ +export async function getLicenseOrUndefined(): Promise { + try { + return (await publicGet('/ee_license')) || undefined + } catch { + return undefined + } +} + +let appCss: Promise | undefined +/** The app-wide stylesheet, which these routes need only off the raw-app path: sign-in, + * errors, skeletons and low-code apps. A raw app's bundle brings its own styles. */ +export function loadAppCss(): Promise { + return (appCss ??= import('$lib/assets/app.css')) +} diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 75de2d7faf..a72d54507e 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -188,6 +188,8 @@ export interface AppEditorProps { // with its author and time. `version` and `head` differ when another deploy landed // beside this one, which is when the out-of-date prompt has something to say. onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } export type App = { diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte new file mode 100644 index 0000000000..49a222148b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte @@ -0,0 +1,150 @@ + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte new file mode 100644 index 0000000000..2a54fe11d4 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte @@ -0,0 +1,73 @@ + + + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte new file mode 100644 index 0000000000..8ebd24455c --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte @@ -0,0 +1,23 @@ + + + + + + + Add downstream step + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte index 880abb685a..57a31e34d8 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte @@ -1,145 +1,36 @@ - - {#snippet trigger()} - - - {/snippet} - + +
+ + {#snippet trigger()} + + + {/snippet} + +
diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 27f6ce6be7..87b085593c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -1,8 +1,11 @@ -
+
onNodePointerEnter(node.id)} + onnodepointerleave={onNodePointerLeave} + onedgepointerenter={onEdgeEnter} + onedgepointerleave={onEdgeLeave} --background-color={false} >
- + + {#if assetsOnlyToggle} + + { + assetsOnly = !e.detail + if (assetsOnlyFolder != undefined) setAssetsOnlyFolder(assetsOnlyFolder, assetsOnly) + }} + /> + {#if withoutPassive.dropped > 0} + { + showPassiveReads = e.detail + try { + localStorage.setItem(SHOW_PASSIVE_READS_KEY, String(e.detail)) + } catch {} + }} + /> + {/if} + + {/if} {#if showMinimap} + +{/if} + +{#if scheduleWizard} + +{/if} + +{#if onAddPipelineScript} + +{/if} + + - + { @@ -2467,7 +2550,12 @@ addEventListener('message', function (e) { {onTakeLatest} {draftBaseVersion} {onRestore} - {onSavedNewAppPath} + onSavedNewAppPath={(newAppPath) => { + if (opWorkspace && $userStore?.email) { + movePreviewSdkConsent($userStore.email, opWorkspace, path, newAppPath) + } + onSavedNewAppPath?.(newAppPath) + }} {policy} {diffDrawer} {newApp} @@ -2484,6 +2572,7 @@ addEventListener('message', function (e) { {getBundle} {onNavigate} {onDeploy} + {onDetails} {onResetToDeployed} {loadedFromDraft} {othersDraftsCount} @@ -2496,6 +2585,7 @@ addEventListener('message', function (e) { sidebarCollapsed={sidebarCollapsed.val} onToggleSidebar={() => (sidebarCollapsed.val = !sidebarCollapsed.val)} {condensedHeader} + {ownsPageHeader} /> {/key} {/if} - {#if buildError} + {#if sdkPrompt} + +
+ {#if sdkPrompt.declined} +
+

+ The preview is blocked until you grant the permissions this app declares. +

+ +
+ {:else} + sdkPrompt && (sdkPrompt.declined = true)} + /> + {/if} +
+ {:else if buildError} + + + {#if !phoneTopbar} {#snippet fallback()} -
-
+ } + ] + : []) + ] + : undefined} + > + Deploy + +{/snippet} diff --git a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte index 557b2a13ad..9eced2d0e6 100644 --- a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte @@ -1,6 +1,6 @@ - +{#await runnerModule then { default: Runner }} + +{:catch} + {#if requestRefused} + +
+ Could not load the app runtime. Reload the page to try again. +
+ {/if} +{/await} {#if iframeSrc} + {/if} diff --git a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte index 1f28cd02c0..28940eb02a 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte @@ -7,16 +7,19 @@ let { scopes, onContinue, - onDecline + onDecline, + declineLabel = 'Open without granting' }: { /** Scopes the app policy declares for its frontend SDK token. */ scopes: string[] /** Fired when the viewer accepts; `dontAskAgain` persists the consent for * this app path so the prompt is skipped until the declared scopes grow. */ onContinue: (dontAskAgain: boolean) => void - /** Fired when the viewer declines: the app still renders, its frontend code - * just gets no token (SDK calls fail). */ + /** Fired when the viewer declines; no token is minted. */ onDecline: () => void + /** Says what declining does where the prompt is shown: the deployed app + * still renders tokenless, the editor preview stays blocked. */ + declineLabel?: string } = $props() let dontAskAgain = $state(false) @@ -34,17 +37,17 @@
    {#each scopes as scope (scope)}
  • - {sdkScopeLabel(scope)} - {#if sdkScopeDescription(scope)} - — {sdkScopeDescription(scope)} - {/if} + {sdkScopeLabel(scope)}{#if sdkScopeDescription(scope)}: {sdkScopeDescription(scope)}{/if}
    • {/each}
- +
diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.test.ts b/frontend/src/lib/components/raw_apps/sdkScopes.test.ts index 68aba9c545..f72918e311 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.test.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from 'vitest' -import { hasStoredSdkConsent, storeSdkConsent } from './sdkScopes' +import { hasStoredSdkConsent, movePreviewSdkConsent, storeSdkConsent } from './sdkScopes' describe('stored frontend SDK consent', () => { beforeEach(() => localStorage.clear()) @@ -20,6 +20,17 @@ describe('stored frontend SDK consent', () => { expect(hasStoredSdkConsent('a@w.dev', 'other', 'u/a/app', ['users:read'])).toBe(false) }) + it('keeps the editor preview and the deployed app apart, and moves preview consent', () => { + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read']) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'], true)).toBe(false) + storeSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'])).toBe(false) + + movePreviewSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', 'u/a/real') + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/real', ['users:read'], true)).toBe(true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true)).toBe(false) + }) + it('treats unreadable storage as no consent', () => { localStorage.setItem('wm_sdk_consent:a@w.dev:ws:u/a/app', 'not json') expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(false) diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index b0177bb660..238f1b7278 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -52,8 +52,17 @@ export function sdkScopeDescription(scope: string): string | undefined { // Keyed by viewer as well as app: this localStorage lives on the shared embedder // origin, so without the viewer one person's "do not ask again" would silently // suppress the prompt for the next person to use the same browser profile. -function sdkConsentKey(viewer: string, workspace: string, path: string): string { - return `wm_sdk_consent:${viewer}:${workspace}:${path}` +// `preview` keeps the editor's consent apart from the deployed app's: the editor +// runs a draft nobody may have reviewed, so trust given to one is not trust given +// to the other. +function sdkConsentKey(viewer: string, workspace: string, path: string, preview: boolean): string { + return `wm_sdk_consent:${preview ? 'p:' : ''}${viewer}:${workspace}:${path}` +} + +/** True when `approved` (a consent the viewer gave) covers every scope the app now + * declares. Fewer scopes need no new consent; one more does. */ +export function sdkConsentCovers(approved: unknown, scopes: string[]): boolean { + return Array.isArray(approved) && scopes.every((s) => approved.includes(s)) } /** True when a previously stored "do not ask again" consent covers every @@ -62,13 +71,14 @@ export function hasStoredSdkConsent( viewer: string, workspace: string, path: string, - scopes: string[] + scopes: string[], + preview = false ): boolean { try { const stored = JSON.parse( - localStorage.getItem(sdkConsentKey(viewer, workspace, path)) ?? 'null' + localStorage.getItem(sdkConsentKey(viewer, workspace, path, preview)) ?? 'null' ) - return Array.isArray(stored) && scopes.every((s) => stored.includes(s)) + return sdkConsentCovers(stored, scopes) } catch (_) { return false } @@ -78,9 +88,29 @@ export function storeSdkConsent( viewer: string, workspace: string, path: string, - scopes: string[] + scopes: string[], + preview = false ): void { try { - localStorage.setItem(sdkConsentKey(viewer, workspace, path), JSON.stringify(scopes)) + localStorage.setItem(sdkConsentKey(viewer, workspace, path, preview), JSON.stringify(scopes)) + } catch (_) {} +} + +/** Follows the editor's app to its new path (first deploy of a draft, or a rename), + * so a stored "do not ask again" consent neither is asked again nor stays behind + * under the old path. A plain Continue is not stored and does not carry over. */ +export function movePreviewSdkConsent( + viewer: string, + workspace: string, + from: string, + to: string +): void { + if (from === to) return + try { + const fromKey = sdkConsentKey(viewer, workspace, from, true) + const stored = localStorage.getItem(fromKey) + if (stored === null) return + localStorage.setItem(sdkConsentKey(viewer, workspace, to, true), stored) + localStorage.removeItem(fromKey) } catch (_) {} } diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts index bdbf62ef8a..3070572785 100644 --- a/frontend/src/lib/components/raw_apps/utils.ts +++ b/frontend/src/lib/components/raw_apps/utils.ts @@ -92,6 +92,8 @@ export type RawAppPreviewLogs = { /** Files were sent to the UI Builder and its build has not reported back in time. */ buildPending: boolean buildLogs: string + /** The app is held back until the user answers its SDK permission prompt, so it is not running. */ + sdkConsentPending: boolean } export type RawAppRuntimeLogRequester = (limit: number) => Promise @@ -292,6 +294,9 @@ export function unsandboxedRawAppHtml( App + +
@@ -299,6 +438,31 @@ f/{path} · data pipeline + {#if workspaceId} + + {/if} +
+ {#if deployErrors.size > 0} + + {/if} + {#if pendingCount > 0} + + {/if}
+ {#key itemPath} + + {/key} {/await} {:else} {#await import('./RawAppEditorView.svelte')} @@ -365,6 +368,7 @@ path={itemPath} {workspaceId} {onNavigate} + onSeeDetails={seeDetails} {isActiveSession} {active} {newRawApp} diff --git a/frontend/src/lib/components/sessions/RawAppEditorView.svelte b/frontend/src/lib/components/sessions/RawAppEditorView.svelte index 137055204d..de05359bd5 100644 --- a/frontend/src/lib/components/sessions/RawAppEditorView.svelte +++ b/frontend/src/lib/components/sessions/RawAppEditorView.svelte @@ -33,6 +33,7 @@ path, workspaceId, onNavigate, + onSeeDetails, isActiveSession = true, active = true, newRawApp = false @@ -41,6 +42,9 @@ path: string workspaceId: string onNavigate?: (item: WorkspaceItem) => void + /** `Exit & see details` — flips this tab to the item's deployed view rather than + * navigating out of the session. */ + onSeeDetails?: (e: { path: string }) => void /** Forwarded to SessionEditorTarget — only the visible session claims the * workspace's single live-editor slot. */ isActiveSession?: boolean @@ -357,6 +361,7 @@ newApp={!cell.saved.val || cell.saved.val.no_deployed === true} {diffDrawer} {onNavigate} + onDetails={onSeeDetails} condensedHeader={true} onResetToDeployed={reloadDeployed} onDeploy={(e) => runtime.itemDeployed(workspaceId, 'raw_app', path, e.path)} diff --git a/frontend/src/lib/components/sessions/SessionChangesBar.svelte b/frontend/src/lib/components/sessions/SessionChangesBar.svelte index 140300ea30..3929b78ca9 100644 --- a/frontend/src/lib/components/sessions/SessionChangesBar.svelte +++ b/frontend/src/lib/components/sessions/SessionChangesBar.svelte @@ -22,7 +22,7 @@ import { badgeCounts, badgeOf, buildDeployItems, type DeployItem } from './sessionDeployModel' import { useExistingMaskKeys } from './sessionDeployModel.svelte' import { previewTargetForDeployKind } from './sessionPreviewTabs.svelte' - import { pipelineFolderFromBundlePath } from '$lib/pipelinePaths' + import { PIPELINE_DRAFT_KIND, pipelineFolderFromBundlePath } from '$lib/pipelinePaths' import type { PreviewTarget } from './previewRouter' import JobsSegment from '$lib/components/copilot/chat/JobsSegment.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' @@ -378,6 +378,11 @@ onDataChanged={refreshDock} onItemDeployed={(item) => void runtime?.manager.renameModifiedItem(item.draftKind, item.path, item.displayPath)} - onItemDiscarded={(item) => void runtime?.manager.removeModifiedItem(item.draftKind, item.path)} + onItemDiscarded={(item) => { + void runtime?.manager.removeModifiedItem(item.draftKind, item.path) + const folder = pipelineFolderFromBundlePath(item.path) + if (item.draftKind === PIPELINE_DRAFT_KIND && folder) void runtime?.forgetPipelineDrafts(folder) + }} + deployPipeline={runtime?.deployPipeline} /> {/if} diff --git a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte index fe5ab2d199..d921e435b0 100644 --- a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte @@ -2,10 +2,10 @@ import WorkspaceDiffDrawer from './WorkspaceDiffDrawer.svelte' import { ArrowRight, GitFork, Pencil } from 'lucide-svelte' import { userWorkspaces } from '$lib/stores' - import Badge from '$lib/components/common/badge/Badge.svelte' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { useSessionDeployModel } from './sessionDeployModel.svelte' import type { DeployItem } from './sessionDeployModel' + import type { DeployResult } from '$lib/utils_workspace_deploy' // Session "Edits" drawer. Builds the deploy model over the session // workspace's drafts and hands it to WorkspaceDiffDrawer, which renders the @@ -19,7 +19,8 @@ keys, onDataChanged, onItemDeployed, - onItemDiscarded + onItemDiscarded, + deployPipeline }: { workspaceId: string parentWorkspaceId?: string @@ -36,6 +37,8 @@ * drops it. */ onItemDeployed?: (item: DeployItem) => void onItemDiscarded?: (item: DeployItem) => void + /** Forwarded to the deploy model: deploys a pipeline folder's draft bundle. */ + deployPipeline?: (folder: string) => Promise } = $props() const isFork = $derived(!!parentWorkspaceId) @@ -51,7 +54,8 @@ mask: keys, onDataChanged, onItemDeployed, - onItemDiscarded + onItemDiscarded, + deployPipeline })) // Editor URL for a row (every item lives in the session workspace). @@ -95,7 +99,7 @@ {ws?.name ?? workspaceId} {#if ws?.is_dev_workspace} - {devBadgeText(ws.dev_workspace_label)} + {/if} diff --git a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte index 9a7fb20bd6..a14990583d 100644 --- a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte +++ b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte @@ -69,7 +69,7 @@ iconOnly={isCollapsed} tooltip={isCollapsed ? 'Workspace' : undefined} size="sm" - class="w-full justify-center" + class="w-full justify-center whitespace-nowrap" onActivate={onNavActivate} /> {/snippet} diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index 9707b537b4..d4f7fa66ab 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -2,7 +2,7 @@ import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import Checkbox from '$lib/components/common/checkbox/Checkbox.svelte' - import { Badge, Button } from '$lib/components/common' + import { Button } from '$lib/components/common' import { Archive, ArchiveRestore, @@ -67,7 +67,7 @@ import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' import { page } from '$app/state' import { base } from '$app/paths' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { dateBucket, GROUP_BY_OPTIONS, @@ -1220,9 +1220,12 @@ {group.label} {#if groupWs?.is_dev_workspace} - - {devBadgeText(groupWs.dev_workspace_label)} - + {/if} {#if groupWsId}
\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -1189,7 +1274,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -1605,21 +1690,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1675,6 +1762,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2167,10 +2262,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2343,6 +2442,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -3432,7 +3543,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, or Cloudflare's\\nJev-compatible Clef) that answers typed questions about a state with calibrated\\nprobabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands @@ -3479,7 +3590,7 @@ app related commands - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - - \`--schema \` - Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist. + - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. - \`--no-open-in-desktop\` - Do not prompt to open the new app in Claude Desktop. - \`app generate-agents [app_folder:string]\` - regenerate AGENTS.md and DATATABLES.md from remote workspace @@ -4163,6 +4274,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml index 70a955d16d..5d5b0f8a0d 100644 --- a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml @@ -84,7 +84,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -95,7 +96,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml index d21d006e8d..e57c92ebc3 100644 --- a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml @@ -71,7 +71,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -82,7 +83,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml index 1d4e81ac2c..79786fc8db 100644 --- a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml @@ -54,7 +54,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -65,7 +66,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml index f35b33ad74..6755d6545c 100644 --- a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml @@ -94,7 +94,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -105,7 +106,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml index f484366783..b428fbf390 100644 --- a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml @@ -133,7 +133,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -144,7 +145,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml index 09217692e5..6549ce1b10 100644 --- a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml @@ -135,7 +135,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -146,7 +147,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml index 2801cc9a73..f167e2965f 100644 --- a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml @@ -96,7 +96,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -107,7 +108,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml index 239d568fcc..e7620683e6 100644 --- a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml @@ -70,7 +70,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -81,7 +82,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml index be196a7bc3..79cca28595 100644 --- a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml @@ -63,7 +63,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -74,7 +75,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/schedule.schema.yaml b/system_prompts/auto-generated/schemas/schedule.schema.yaml index d0ea85d6fe..7097be7d25 100644 --- a/system_prompts/auto-generated/schemas/schedule.schema.yaml +++ b/system_prompts/auto-generated/schemas/schedule.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml index 4f41c32421..32d6400973 100644 --- a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml index 780b0f2245..8184d3815a 100644 --- a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml @@ -172,7 +172,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -183,7 +184,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 0517cf1897..14ca869180 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1567,21 +1567,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1637,6 +1639,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2129,10 +2139,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2305,6 +2319,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 97db78b3d4..13a93ed063 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -60,10 +60,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -236,6 +240,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index 55f59b1f36..55df35ccf7 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -124,21 +124,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -194,6 +196,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index df19b6dac2..3f41b5cfba 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -48,7 +48,7 @@ app related commands - `--path ` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - `--framework ` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. - `--datatable ` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - - `--schema ` - Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist. + - `--schema ` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - `--overwrite` - Overwrite the target directory if it already exists, without prompting. - `--no-open-in-desktop` - Do not prompt to open the new app in Claude Desktop. - `app generate-agents [app_folder:string]` - regenerate AGENTS.md and DATATABLES.md from remote workspace @@ -732,6 +732,8 @@ user related commands **Subcommands:** +- `user list` - List the users of the workspace + - `--json` - Output as JSON (for piping to jq) - `user add [password:string]` - Create a user - `--superadmin` - Specify to make the new user superadmin. - `--company ` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index fe9f57c55d..2f7cd52d5f 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -11,47 +11,53 @@ This guide covers raw apps from the terminal: scaffolding via `wmill app new`, t **You — the AI agent — create the app yourself by running `wmill app new` with the right flags. Do NOT tell the user to "run `wmill app new` and follow the prompts" or wait for them to do it.** The bare `wmill app new` is an interactive wizard that hangs waiting for stdin in any non-TTY context (which includes you). Always pass flags. -### Step 1 — Gather the three required values by asking the user +### Step 1 — Settle the setup -You need three things to run the command: +You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. `f/folder/my_app` or `u/username/my_app` 3. **framework** — one of `react19` (recommended), `react18`, `svelte5`, `vue` -If the user's request did not supply *every* one of these explicitly, ask. Do not guess values, do not invent paths, do not pick a framework on the user's behalf, do not "just use react19 because it's the default". +**summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. -Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group all missing fields into a single round-trip so the user answers them at once: +**framework**: never ask. Use the one the user named, otherwise the one the project's existing apps use, otherwise `react19` (see "Starting a new app" in the authoring guide). + +**data setup**: when the app has to store data and the user did not say where, ask which data table, which schema and which tables, as "Data setup" in the authoring guide lays out. `wmill datatable list` gives the data tables to offer. + +Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group everything missing into a single round-trip so the user answers at once: -- For `framework` — multiple-choice with the four allowed values; mark `react19` as `(Recommended)` and put it first. - For `summary` and `path` — provide one or two example values as multiple-choice options (the user can pick "Other" to type a free-form answer). +- For the data setup — the choices the authoring guide lists, the default first. -Only proceed once you have concrete values for all three. If the user replies with something ambiguous, ask again rather than guessing. +Only proceed once every value is concrete. If the user replies with something ambiguous, ask again rather than guessing. ### Step 2 — Run the command yourself -Once you have summary + path + framework, run it: +Once the setup is settled, run it: ```bash wmill app new \ --summary "Customer dashboard" \ --path f/sales/dashboard \ - --framework react19 + --framework react19 \ + --datatable main \ + --schema app1 ``` -That's the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of `--summary`/`--path`/`--framework` puts the command in non-interactive mode. +`--summary`, `--path` and `--framework` are the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of them puts the command in non-interactive mode, so the data setup only reaches the app through the flags below. ### Optional flags -Layer these in only when the user asked for them: - | Flag | When to add it | |---|---| -| `--datatable ` | The user wants this app wired to a specific Windmill datatable. Without it, the app is created with no datatable. | -| `--schema ` | Together with `--datatable`. Creates the schema with `CREATE SCHEMA IF NOT EXISTS` if it doesn't already exist. | +| `--datatable ` | The app stores data: the data table settled in step 1. Without it, the app is created with no datatable. | +| `--schema ` | Together with `--datatable`, when the app's tables go in a schema, new or existing. If the schema doesn't exist yet, writes a `sql_to_apply/` migration with `CREATE SCHEMA IF NOT EXISTS`, applied like any other migration (see "SQL Migrations" below). Leave it out for `public`. | | `--overwrite` | The target directory already exists and the user said it's OK to replace. Without it, non-interactive mode aborts with an error so you don't clobber existing work. | | `--no-open-in-desktop` | Already implied in non-interactive mode; only needed if you're somehow running interactively. | +Existing tables the app reuses have no flag: list them under `data.tables` in `raw_app.yaml` (see "Data tables" below). + ### Step 3 — Offer the visual preview After `wmill app new` and any initial edits to `App.tsx` / `index.tsx`, **offer** to open the visual preview as a one-sentence next step (e.g. "Want me to open the visual preview?"). Don't auto-open — opening the dev page has side effects (browser window, possibly a `launch.json` entry when an embedded preview tool is in play) the user should consent to. @@ -62,8 +68,9 @@ For apps the preview command runs from the app folder (`cd __raw_app & - ❌ Running `wmill app new` with no flags (the prompt will hang). - ❌ Telling the user to "run `wmill app new` and follow the prompts" — that's a step backwards from what you can do directly. -- ❌ Inventing a path/summary/framework instead of asking the user. -- ❌ Defaulting to `react19` because the user didn't say — even sensible defaults must be confirmed. +- ❌ Inventing a path or summary instead of asking the user. +- ❌ Asking which framework to use — `react19` is the default when nothing says otherwise. +- ❌ Creating a schema or tables for a new app before the user has said where its data goes. - ❌ Passing `--overwrite` automatically when the directory exists — confirm with the user first. ### Interactive (only when a human is at the terminal) @@ -250,6 +257,38 @@ A raw app has three logical parts: - **Backend runnables** — server-side scripts the frontend calls, each addressed by a unique key. - **Data** — optional whitelisted datatables (managed PostgreSQL) that the backend runnables can query. The frontend never queries the database directly; backend runnables are the only bridge. +## Starting a new app + +A new app gets the setup Windmill's own new-app dialog gives one: a framework, then where its data lives. + +### Framework + +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. + +### Data setup + +When a new app has to store data and the user did not say where, **ask before creating anything**: no app, no schema, no table. Where the data lives is the user's decision, and tables created in the wrong place stay there. Look up the workspace's data tables first so every choice you offer exists, then ask, as proposed answers rather than open questions: + +1. **Data table**: which one to use, the default first (`main` when it exists, otherwise the first listed). Skip this one when the workspace has a single data table. +2. **Schema**: where the app's new tables go. + - A **new schema**, the usual choice, offered first. Suggest the first unused name among `app1`, `app2`, … + - An **existing schema** of that data table, by name. + - **None**: the tables go in `public`. +3. **Tables**: only when that data table already holds tables the app could use, whether to **reuse** them (name them) or **create new ones**. + +Do not ask when: + +- the user already said where the data goes. A part they left out takes its default above: the default data table, a new `appN` schema, new tables; +- the app, or its saved draft, already carries a data config (`data.datatable`, `data.schema` or `data.tables`). That config is the answer; +- the app stores nothing. + +With no data table in the workspace there is nothing to choose: say that one has to be configured in the workspace settings before the app can store data. + +A question left unanswered or dismissed is not a go-ahead for the defaults: stop there and say what is still needed. + +Then build what was chosen: the data table and schema become the app's `data.datatable` and `data.schema`, the tables it uses, reused or new, go in `data.tables`, and a new schema is created (`CREATE SCHEMA IF NOT EXISTS`) before the tables that go in it. +`wmill app new --datatable --schema ` records both and, for a schema that does not exist yet, writes the migration in `sql_to_apply/` that creates it: nothing exists in the database until `wmill app dev` applies that migration, so the app's table migrations come after it. + ## Frontend ### Entrypoint diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 66bf3d0fa4..4b0889c3d8 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -109,6 +109,7 @@ These are strict Windmill schema rules. Follow them exactly. - `preprocessor_module` and `failure_module` only support `script` or `rawscript` - `preprocessor_module` runs before normal modules and cannot reference `results.*` - `failure_module` can use the `error` object with `error.message`, `error.step_id`, `error.name`, and `error.stack` +- A flow whose `failure_module` runs still ends as failed, unless the handler returns an object with `recover: true`: the flow then ends as a success. This also holds when the failing step is inside a loop, a branch or a subflow. `recover` never changes which steps run; whether a loop or branch carries on past a failure is still decided by its own settings (`skip_failures`, `continue_on_error`) Correct shape: @@ -266,7 +267,7 @@ names, so neither is name-checked at all — leave those summaries as they are. ## AI Decision Modules -An `aidecision` module asks a decision model (TypeSafe's Jev or Cloudflare's Clef) typed questions about a `state` and +An `aidecision` module asks a decision model (TypeSafe's Jev, Cloudflare's Clef or OpenAI's Decisions API) typed questions about a `state` and answers each with calibrated probabilities instead of text. Prefer it over an `aiagent` when the step is a judgment (classify, route, score or a yes/no check) that needs no tools and no free text: it is faster, cheaper, and its answers have a fixed shape. @@ -304,8 +305,9 @@ is faster, cheaper, and its answers have a fixed shape. } ``` -- `provider.kind` is `typesafe` (`model` `jev-latest` unless a version is pinned) or `cloudflare` - (`model` `clef`, or `clef-flash` for faster answers); the resource is of that same type +- `provider.kind` is `typesafe` (`model` `jev-latest` unless a version is pinned), `cloudflare` + (`model` `clef`, or `clef-flash` for faster answers) or `openai` (`model` `gpt-6-luna`, never a + chat model); the resource is of that same type - `state` is the content to evaluate: usually a text, such as the message to classify. To combine several values, pass an object with descriptive keys holding only what the questions need (`({ message: flow_input.message, plan: results.get_account.plan })`); an array of strings also works @@ -319,6 +321,12 @@ is faster, cheaper, and its answers have a fixed shape. - The result is `{ output, model, usage }` with `output` keyed by question name, so a later step reads `results.triage.output.intent.choice`, `results.triage.output.urgency.score` or `results.triage.output.angry.noul > 0.7` +- On `openai` the same `questions` are translated for its Decisions API and the answers read the + same way (`choice`, `score`, `noul`), except that `probabilities` is a list of + `{ value, probability }` (plus `label` on a score) and a score has no `legend`. A question OpenAI + refuses to answer fails the step. A `state` that is + an array of user messages (`[{ role: "user", content: [{ type: "input_text", text }, { type: "input_image", image_url: "data:image/png;base64,..." }] }]`) + is sent as is, which is how an image is evaluated; any other object or array is sent as JSON text ### Branching on the Answers @@ -363,6 +371,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules @@ -674,4 +686,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, or Cloudflare's\nJev-compatible Clef) that answers typed questions about a state with calibrated\nprobabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} diff --git a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md index 064dc600ac..4c339e6222 100644 --- a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md +++ b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md @@ -12,10 +12,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -59,7 +97,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 4b340c0db1..6ad44db068 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -320,21 +320,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -390,6 +392,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 03d40ab044..a5f599a946 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -332,21 +332,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -402,6 +404,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index b15655a6da..ce0c610da7 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -322,21 +322,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -392,6 +394,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index b9c2793f47..4946ed0f6e 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -268,10 +268,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -444,6 +448,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index c615a628f3..ff10f38062 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -22,6 +22,7 @@ These are strict Windmill schema rules. Follow them exactly. - `preprocessor_module` and `failure_module` only support `script` or `rawscript` - `preprocessor_module` runs before normal modules and cannot reference `results.*` - `failure_module` can use the `error` object with `error.message`, `error.step_id`, `error.name`, and `error.stack` +- A flow whose `failure_module` runs still ends as failed, unless the handler returns an object with `recover: true`: the flow then ends as a success. This also holds when the failing step is inside a loop, a branch or a subflow. `recover` never changes which steps run; whether a loop or branch carries on past a failure is still decided by its own settings (`skip_failures`, `continue_on_error`) Correct shape: @@ -184,7 +185,7 @@ names, so neither is name-checked at all — leave those summaries as they are. ## AI Decision Modules -An `aidecision` module asks a decision model (TypeSafe's Jev or Cloudflare's Clef) typed questions about a `state` and +An `aidecision` module asks a decision model (TypeSafe's Jev, Cloudflare's Clef or OpenAI's Decisions API) typed questions about a `state` and answers each with calibrated probabilities instead of text. Prefer it over an `aiagent` when the step is a judgment (classify, route, score or a yes/no check) that needs no tools and no free text: it is faster, cheaper, and its answers have a fixed shape. @@ -222,8 +223,9 @@ is faster, cheaper, and its answers have a fixed shape. } ``` -- `provider.kind` is `typesafe` (`model` `jev-latest` unless a version is pinned) or `cloudflare` - (`model` `clef`, or `clef-flash` for faster answers); the resource is of that same type +- `provider.kind` is `typesafe` (`model` `jev-latest` unless a version is pinned), `cloudflare` + (`model` `clef`, or `clef-flash` for faster answers) or `openai` (`model` `gpt-6-luna`, never a + chat model); the resource is of that same type - `state` is the content to evaluate: usually a text, such as the message to classify. To combine several values, pass an object with descriptive keys holding only what the questions need (`({ message: flow_input.message, plan: results.get_account.plan })`); an array of strings also works @@ -237,6 +239,12 @@ is faster, cheaper, and its answers have a fixed shape. - The result is `{ output, model, usage }` with `output` keyed by question name, so a later step reads `results.triage.output.intent.choice`, `results.triage.output.urgency.score` or `results.triage.output.angry.noul > 0.7` +- On `openai` the same `questions` are translated for its Decisions API and the answers read the + same way (`choice`, `score`, `noul`), except that `probabilities` is a list of + `{ value, probability }` (plus `label` on a score) and a score has no `legend`. A question OpenAI + refuses to answer fails the step. A `state` that is + an array of user messages (`[{ role: "user", content: [{ type: "input_text", text }, { type: "input_image", image_url: "data:image/png;base64,..." }] }]`) + is sent as is, which is how an image is evaluated; any other object or array is sent as JSON text ### Branching on the Answers @@ -281,6 +289,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index 01e9797e32..59924faf41 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -7,10 +7,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -56,7 +94,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/system_prompts/base/raw-app-cli.md b/system_prompts/base/raw-app-cli.md index 0ff11a2b74..59ef0acb58 100644 --- a/system_prompts/base/raw-app-cli.md +++ b/system_prompts/base/raw-app-cli.md @@ -6,47 +6,53 @@ This guide covers raw apps from the terminal: scaffolding via `wmill app new`, t **You — the AI agent — create the app yourself by running `wmill app new` with the right flags. Do NOT tell the user to "run `wmill app new` and follow the prompts" or wait for them to do it.** The bare `wmill app new` is an interactive wizard that hangs waiting for stdin in any non-TTY context (which includes you). Always pass flags. -### Step 1 — Gather the three required values by asking the user +### Step 1 — Settle the setup -You need three things to run the command: +You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. `f/folder/my_app` or `u/username/my_app` 3. **framework** — one of `react19` (recommended), `react18`, `svelte5`, `vue` -If the user's request did not supply *every* one of these explicitly, ask. Do not guess values, do not invent paths, do not pick a framework on the user's behalf, do not "just use react19 because it's the default". +**summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. -Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group all missing fields into a single round-trip so the user answers them at once: +**framework**: never ask. Use the one the user named, otherwise the one the project's existing apps use, otherwise `react19` (see "Starting a new app" in the authoring guide). + +**data setup**: when the app has to store data and the user did not say where, ask which data table, which schema and which tables, as "Data setup" in the authoring guide lays out. `wmill datatable list` gives the data tables to offer. + +Use whichever interactive question facility your runtime provides — a structured multi-choice tool if available, otherwise plain chat — and group everything missing into a single round-trip so the user answers at once: -- For `framework` — multiple-choice with the four allowed values; mark `react19` as `(Recommended)` and put it first. - For `summary` and `path` — provide one or two example values as multiple-choice options (the user can pick "Other" to type a free-form answer). +- For the data setup — the choices the authoring guide lists, the default first. -Only proceed once you have concrete values for all three. If the user replies with something ambiguous, ask again rather than guessing. +Only proceed once every value is concrete. If the user replies with something ambiguous, ask again rather than guessing. ### Step 2 — Run the command yourself -Once you have summary + path + framework, run it: +Once the setup is settled, run it: ```bash wmill app new \ --summary "Customer dashboard" \ --path f/sales/dashboard \ - --framework react19 + --framework react19 \ + --datatable main \ + --schema app1 ``` -That's the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of `--summary`/`--path`/`--framework` puts the command in non-interactive mode. +`--summary`, `--path` and `--framework` are the minimum. The datatable wizard and the "Open in Claude Desktop?" prompt are skipped silently because passing any of them puts the command in non-interactive mode, so the data setup only reaches the app through the flags below. ### Optional flags -Layer these in only when the user asked for them: - | Flag | When to add it | |---|---| -| `--datatable ` | The user wants this app wired to a specific Windmill datatable. Without it, the app is created with no datatable. | -| `--schema ` | Together with `--datatable`. Creates the schema with `CREATE SCHEMA IF NOT EXISTS` if it doesn't already exist. | +| `--datatable ` | The app stores data: the data table settled in step 1. Without it, the app is created with no datatable. | +| `--schema ` | Together with `--datatable`, when the app's tables go in a schema, new or existing. If the schema doesn't exist yet, writes a `sql_to_apply/` migration with `CREATE SCHEMA IF NOT EXISTS`, applied like any other migration (see "SQL Migrations" below). Leave it out for `public`. | | `--overwrite` | The target directory already exists and the user said it's OK to replace. Without it, non-interactive mode aborts with an error so you don't clobber existing work. | | `--no-open-in-desktop` | Already implied in non-interactive mode; only needed if you're somehow running interactively. | +Existing tables the app reuses have no flag: list them under `data.tables` in `raw_app.yaml` (see "Data tables" below). + ### Step 3 — Offer the visual preview After `wmill app new` and any initial edits to `App.tsx` / `index.tsx`, **offer** to open the visual preview as a one-sentence next step (e.g. "Want me to open the visual preview?"). Don't auto-open — opening the dev page has side effects (browser window, possibly a `launch.json` entry when an embedded preview tool is in play) the user should consent to. @@ -57,8 +63,9 @@ For apps the preview command runs from the app folder (`cd __raw_app & - ❌ Running `wmill app new` with no flags (the prompt will hang). - ❌ Telling the user to "run `wmill app new` and follow the prompts" — that's a step backwards from what you can do directly. -- ❌ Inventing a path/summary/framework instead of asking the user. -- ❌ Defaulting to `react19` because the user didn't say — even sensible defaults must be confirmed. +- ❌ Inventing a path or summary instead of asking the user. +- ❌ Asking which framework to use — `react19` is the default when nothing says otherwise. +- ❌ Creating a schema or tables for a new app before the user has said where its data goes. - ❌ Passing `--overwrite` automatically when the directory exists — confirm with the user first. ### Interactive (only when a human is at the terminal) diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index 56c6e900d4..03bde6ec71 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -10,6 +10,46 @@ A raw app has three logical parts: - **Backend runnables** — server-side scripts the frontend calls, each addressed by a unique key. - **Data** — optional whitelisted datatables (managed PostgreSQL) that the backend runnables can query. The frontend never queries the database directly; backend runnables are the only bridge. +## Starting a new app + +A new app gets the setup Windmill's own new-app dialog gives one: a framework, then where its data lives. + +### Framework + +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. + +### Data setup + +When a new app has to store data and the user did not say where, **ask before creating anything**: no app, no schema, no table. Where the data lives is the user's decision, and tables created in the wrong place stay there. Look up the workspace's data tables first so every choice you offer exists, then ask, as proposed answers rather than open questions: + +1. **Data table**: which one to use, the default first (`main` when it exists, otherwise the first listed). Skip this one when the workspace has a single data table. +2. **Schema**: where the app's new tables go. + - A **new schema**, the usual choice, offered first. Suggest the first unused name among `app1`, `app2`, … + - An **existing schema** of that data table, by name. + - **None**: the tables go in `public`. +3. **Tables**: only when that data table already holds tables the app could use, whether to **reuse** them (name them) or **create new ones**. + +Do not ask when: + +- the user already said where the data goes. A part they left out takes its default above: the default data table, a new `appN` schema, new tables; +- the app, or its saved draft, already carries a data config (`data.datatable`, `data.schema` or `data.tables`). That config is the answer; +- the app stores nothing. + +With no data table in the workspace there is nothing to choose: say that one has to be configured in the workspace settings before the app can store data. + +A question left unanswered or dismissed is not a go-ahead for the defaults: stop there and say what is still needed. + +Then build what was chosen: the data table and schema become the app's `data.datatable` and `data.schema`, the tables it uses, reused or new, go in `data.tables`, and a new schema is created (`CREATE SCHEMA IF NOT EXISTS`) before the tables that go in it. + +`wmill app new --datatable --schema ` records both and, for a schema that does not exist yet, writes the migration in `sql_to_apply/` that creates it: nothing exists in the database until `wmill app dev` applies that migration, so the app's table migrations come after it. + + + +### An app that arrives already set up + +An app the user just started from the new-app dialog is already set up: its files are the framework template it was started with, and a `data` config it carries (`data.datatable`, `data.schema` or `data.tables`) is the data setup it was started with. Build that app in place, with that data config, and in that framework unless the user now names another. Do not ask for either, and do not create a second app. Only when it carries no data config and has to store data is the data setup still to ask. + + ## Frontend ### Entrypoint diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index e1ad956dc4..41a8eac37e 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -22,6 +22,6 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 8b51f75dfc..f927acd26e 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -46,7 +46,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" # Build default export by combining client utilities + services # This preserves backward compatibility for `import wmill from "windmill-client"` @@ -63,6 +63,7 @@ import { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, @@ -153,6 +154,7 @@ const wmill = { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 7e84320346..3f7b03117a 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -11,6 +11,7 @@ import { UserService, KafkaTriggerService, } from "./services.gen"; +import type { Retry } from "./types.gen"; import { OpenAPI } from "./core/OpenAPI"; import { isSuspendSignal, stepErrorMarker, taskErrorFromMarker } from "./wacError"; // import type { DenoS3LightClientSettings } from "./index"; @@ -420,12 +421,26 @@ export async function runScriptAsync( return _runScriptAsyncInternal(path, hash_, args, scheduledInSeconds, tag); } +/** Options for {@link runScriptByPathAsync} and {@link runScriptByHashAsync}, passed in + * place of `scheduledInSeconds`; a positional `tag` is then ignored. */ +export interface RunScriptAsyncOptions { + /** Schedule execution for a future time (in seconds) */ + scheduledInSeconds?: number; + /** Override the worker tag the job runs on */ + tag?: string; + /** Re-run the job when it fails, with the same policy a schedule takes, e.g. + * `{ constant: { attempts: 3, seconds: 10 } }`. The returned id is the first + * attempt's; each retry is a job of its own, recorded as a child of it. */ + retry?: Retry; +} + async function _runScriptAsyncInternal( path: string | null = null, hash_: string | null = null, args: Record | null = null, scheduledInSeconds: number | null = null, - tag: string | null = null + tag: string | null = null, + retry: Retry | null = null ): Promise { // Create a script job and return its job id. args = args || {}; @@ -439,6 +454,10 @@ async function _runScriptAsyncInternal( params["tag"] = tag; } + if (retry) { + params["retry"] = JSON.stringify(retry); + } + let parentJobId = getEnv("WM_JOB_ID"); if (parentJobId !== undefined) { params["parent_job"] = parentJobId; @@ -475,34 +494,47 @@ async function _runScriptAsyncInternal( * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByPathAsync( path: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(path, null, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(path, null, args, o.scheduledInSeconds, o.tag, o.retry); } /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByHashAsync( hash_: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(null, hash_, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(null, hash_, args, o.scheduledInSeconds, o.tag, o.retry); +} + +function asyncRunOptions( + scheduledInSeconds: number | null | RunScriptAsyncOptions, + tag: string | null +): RunScriptAsyncOptions { + return typeof scheduledInSeconds === "object" && scheduledInSeconds !== null + ? scheduledInSeconds + : { scheduledInSeconds: scheduledInSeconds ?? undefined, tag: tag ?? undefined }; } /** @@ -692,6 +724,19 @@ export async function getProgress(jobId?: any): Promise { }); } +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +export async function workerIsDraining(jobId?: string): Promise { + return await JobService.workerIsDraining({ + id: jobId ?? getEnv("WM_JOB_ID") ?? "NO_JOB_ID", + workspace: getWorkspace(), + }); +} + /** * Set a flow user state * @param key key of the state diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 488d265310..45306830f5 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.826.0", + "version": "1.828.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index a517f1ff0f..98caf3895a 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.826.0", + "version": "1.828.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 47ce201f9f..b08184bb19 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.826.0 +1.828.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 0c496bc161..064996f393 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.828.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index da525da3b8..7e8c7a641d 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.828.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",