Merge remote-tracking branch 'origin/main' into dependabot/cargo/backend/windmill-duckdb-ffi-internal/cargo-e3be83c1e1

.github/workflows/ai-evals-scheduled.yml

@@ -15,8 +15,8 @@ on:
         description: "Space-separated modes (global, cli, flow, app, script)"
         default: "global cli"
       models:
-        description: "Space-separated model aliases (bun run cli -- models); cli runs only the Claude ones"
-        default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
+        description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
+        default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro"
       runs:
         description: "Runs per case"
         default: "3"
@@ -28,7 +28,7 @@ concurrency:
   group: ai-evals-scheduled-${{ github.ref }}
 
 env:
-  MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }}
+  MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }}
   RUNS: ${{ inputs.runs || '3' }}
   REASONING: ${{ inputs.reasoning }}
   # Routes on the cloud instance are prefixed with their workspace.
@@ -40,17 +40,28 @@ jobs:
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
+      - uses: actions/checkout@v4
+        with:
+          sparse-checkout: ai_evals/core
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: 1.4.0
+
       - id: matrix
         env:
           MODES: ${{ inputs.modes || 'global cli' }}
-        # cli drives the Claude Agent SDK, so it only takes the Claude aliases
-        # (`cli` entries in ai_evals/core/models.ts).
+        # cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts.
         run: |
-          echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" '
+          cli_models=$(bun -e '
+            import { EVAL_MODELS } from "./ai_evals/core/models.ts";
+            console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli)
+              .flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));')
+          echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" '
             def words: split(" ") | map(select(. != ""));
             [ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
               | $models | words[]
-              | select($mode != "cli" or test("^(claude-)?(haiku|sonnet|opus)"; "i"))
+              | select($mode != "cli" or (ascii_downcase | IN($cli[])))
               | {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
 
   evals:
@@ -120,6 +131,10 @@ jobs:
         working-directory: ./cli
         run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
 
+      - name: Install Codex CLI
+        if: matrix.mode == 'cli'
+        run: npm install --global @openai/codex@0.159.3
+
       - name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
         working-directory: ./ai_evals
         env:

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "1.826.0"
+  ".": "1.828.0"
 }

CHANGELOG.md

@@ -1,5 +1,48 @@
 # Changelog
 
+## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08)
+
+
+### Features
+
+* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864))
+* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d))
+* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4))
+* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6))
+* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24))
+
+
+### Bug Fixes
+
+* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775))
+* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba))
+* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687))
+
+
+### Performance Improvements
+
+* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996))
+
+## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07)
+
+
+### Features
+
+* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9))
+* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842))
+* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8))
+* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f))
+* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971))
+
+
+### Bug Fixes
+
+* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3))
+* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67))
+* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2))
+* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1))
+* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7))
+
 ## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07)
 
 

ai_evals/README.md

@@ -89,7 +89,21 @@ Notes:
 
 - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
 - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
-- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
+- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there:
+
+  | Agent | Models | Label | Needs |
+  | --- | --- | --- | --- |
+  | Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:<model>` | `ANTHROPIC_API_KEY` |
+  | Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:<model>` | `DEEPSEEK_API_KEY` |
+  | Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:<model>` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` |
+
+  Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no
+  Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace
+  records any command that touches `.agents/skills/<name>/SKILL.md` as a `Skill` call to `<name>`, just
+  before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the
+  `invokes skill … before first mutation` checks compare the same things for both agents. Codex has no
+  turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a
+  Claude Code run does at its cap.
 - the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs
 
 ## Case Format
@@ -164,6 +178,12 @@ the decrypted value, exactly as against a real backend. The chat's read path pas
 Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and
 assert it via `valueExcludes` to catch a leak.
 
+`runtime.userAnswers` scripts the user's side of `askUserQuestion` in a global case: one
+answer per question, in the order they are asked, and a question past the last answer is
+dismissed, so `userAnswers: []` dismisses every question. Without it the tool reports that
+it cannot ask, and the model usually carries on with its own guess, which makes "asks
+before creating anything" impossible to assert.
+
 `toolExpect.toolCallArgs` entries support `sharedByAtLeast: <n>`: at least `n` recorded
 calls to that tool must carry the same non-blank string in the field. Use it for calls that
 have to share an identifier, like two test runs of one chat conversation.

ai_evals/adapters/cli/codex.test.ts

@@ -0,0 +1,71 @@
+import { describe, expect, it } from "bun:test";
+import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex";
+
+describe("unwrapShellCommand", () => {
+  it("strips the login-shell wrapper Codex reports", () => {
+    expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe(
+      "cat .agents/skills/x/SKILL.md"
+    );
+    expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe(
+      `printf 'hi' > "a b"`
+    );
+    expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`);
+    expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push");
+    expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`);
+  });
+});
+
+describe("toToolInvocations", () => {
+  it("records reading a SKILL.md as a Skill call before the command", () => {
+    const tools = toToolInvocations({
+      type: "command_execution",
+      command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`,
+    });
+    expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([
+      ["Skill", { skill: "write-script-bun" }],
+      ["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }],
+    ]);
+  });
+
+  it("records a skill read after a mutation earlier in the same command", () => {
+    const tools = toToolInvocations({
+      type: "command_execution",
+      command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`,
+    });
+    expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]);
+  });
+
+  it("maps a patch to Edit so it counts as a mutation", () => {
+    expect(
+      toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0]
+    ).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } });
+  });
+});
+
+describe("runCodex", () => {
+  it("keeps parallel tool calls in the order they started", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "fake-codex-"));
+    const events = [
+      { type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } },
+      { type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
+      { type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
+      { type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } },
+      { type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } },
+      { type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } },
+    ];
+    const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n");
+    await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`);
+    await chmod(join(dir, "codex"), 0o755);
+    try {
+      const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` });
+      expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]);
+      expect(result.output).toBe("done");
+      expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 });
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

ai_evals/adapters/cli/codex.ts

@@ -0,0 +1,241 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types";
+import { isLikelyMutatingBashCommand } from "./mutation";
+
+export interface AgentRunResult {
+  output: string;
+  toolsUsed: CliToolInvocation[];
+  assistantMessageCount: number;
+  tokenUsage: BenchmarkTokenUsage | null;
+  finalContextTokens: number | null;
+}
+
+interface CodexItem {
+  type?: string;
+  text?: string;
+  command?: string;
+  changes?: { path?: string; kind?: string }[];
+  [key: string]: unknown;
+}
+
+interface CodexEvent {
+  type?: string;
+  item?: CodexItem;
+  usage?: { input_tokens?: number; output_tokens?: number };
+  message?: string;
+  error?: { message?: string };
+}
+
+const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g;
+
+// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary
+// the validators read: shell commands become `Bash`, patches `Edit`, and
+// reading `.agents/skills/<name>/SKILL.md` (how Codex loads a skill) becomes a
+// `Skill` call recorded just before that command.
+export async function runCodex(
+  prompt: string,
+  cwd: string,
+  maxTurns: number,
+  model: string,
+  env: Record<string, string>
+): Promise<AgentRunResult> {
+  // A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run.
+  const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-"));
+  const result: AgentRunResult = {
+    output: "",
+    toolsUsed: [],
+    assistantMessageCount: 0,
+    tokenUsage: null,
+    finalContextTokens: null,
+  };
+  const messages: string[] = [];
+  let failure: string | null = null;
+  let lastError: string | null = null;
+  let stderr = "";
+
+  try {
+    const child = spawn(
+      "codex",
+      [
+        "exec",
+        "--json",
+        "--ephemeral",
+        "--skip-git-repo-check",
+        "--dangerously-bypass-approvals-and-sandbox",
+        "--model",
+        model,
+        "--cd",
+        cwd,
+        "-",
+      ],
+      {
+        cwd,
+        env: {
+          ...env,
+          HOME: home,
+          CODEX_HOME: home,
+          CODEX_API_KEY: env.OPENAI_API_KEY ?? "",
+        },
+        stdio: ["pipe", "pipe", "pipe"],
+      }
+    );
+    // Without this, a missing `codex` binary leaves the stdout loop waiting forever.
+    await new Promise<void>((resolve, reject) => {
+      child.once("spawn", resolve);
+      child.once("error", reject);
+    });
+    child.stdin.on("error", () => {});
+    child.stdin.end(prompt);
+    child.stderr.on("data", (chunk) => {
+      stderr = (stderr + chunk).slice(-4000);
+    });
+    const exited = new Promise<number | null>((resolve, reject) => {
+      child.on("error", reject);
+      child.on("close", resolve);
+    });
+
+    // Codex has no turn cap. A turn starts when the agent issues tool calls with
+    // none in flight, so parallel calls count once, as they do for Claude Code.
+    // Claude Code's cap includes the final reply, so the maxTurns-th tool round is
+    // already over it.
+    // ponytail: parallel calls that finish before the next one starts count as
+    // separate turns; count model calls if Codex ever reports them.
+    let turns = 0;
+    let inFlight = 0;
+    let capped = false;
+    // Parallel calls complete out of order; the trace keeps the order they started in.
+    const slots = new Map<string, number>();
+    const ordered: CliToolInvocation[][] = [];
+    const slotFor = (item: CodexItem) => {
+      const id = typeof item.id === "string" ? item.id : undefined;
+      if (id !== undefined && slots.has(id)) {
+        return slots.get(id)!;
+      }
+      ordered.push([]);
+      if (id !== undefined) {
+        slots.set(id, ordered.length - 1);
+      }
+      return ordered.length - 1;
+    };
+
+    for await (const line of createInterface({ input: child.stdout })) {
+      const event = parseEvent(line);
+      if (!event) {
+        continue;
+      }
+      const item = event.item;
+
+      if (event.type === "item.started" && item && isToolItem(item)) {
+        if (inFlight === 0) {
+          turns += 1;
+        }
+        inFlight += 1;
+        slotFor(item);
+        if (turns >= maxTurns && !capped) {
+          capped = true;
+          child.kill("SIGTERM");
+        }
+      } else if (event.type === "item.completed" && item) {
+        if (item.type === "agent_message") {
+          result.assistantMessageCount += 1;
+          if (typeof item.text === "string") {
+            messages.push(item.text);
+          }
+        } else if (isToolItem(item)) {
+          inFlight = Math.max(0, inFlight - 1);
+          ordered[slotFor(item)] = toToolInvocations(item);
+        }
+      } else if (event.type === "turn.completed" && event.usage) {
+        const prompt = event.usage.input_tokens ?? 0;
+        const completion = event.usage.output_tokens ?? 0;
+        result.tokenUsage = { prompt, completion, total: prompt + completion };
+      } else if (event.type === "turn.failed") {
+        failure = event.error?.message ?? JSON.stringify(event);
+      } else if (event.type === "error") {
+        // Also sent for retries Codex recovers from ("Reconnecting... 1/5").
+        lastError = event.message ?? JSON.stringify(event);
+      }
+    }
+
+    result.toolsUsed = ordered.flat();
+    const code = await exited;
+    if (failure) {
+      throw new Error(`codex: ${failure}`);
+    }
+    // Fail the attempt, as Claude Code does at its cap.
+    if (capped) {
+      throw new Error(`codex: reached maximum number of turns (${maxTurns})`);
+    }
+    if (code !== 0) {
+      throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`);
+    }
+  } finally {
+    await rm(home, { recursive: true, force: true });
+  }
+
+  result.output = messages.join("\n");
+  return result;
+}
+
+export function toToolInvocations(item: CodexItem): CliToolInvocation[] {
+  const timestamp = Date.now();
+
+  if (item.type === "command_execution" && typeof item.command === "string") {
+    const command = unwrapShellCommand(item.command);
+    // A skill read counts as preceding the command only when nothing before it in
+    // the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a
+    // skill-before-first-mutation check.
+    const before: string[] = [];
+    const after: string[] = [];
+    for (const match of command.matchAll(SKILL_FILE_PATTERN)) {
+      const skill = match[1]!;
+      if (before.includes(skill) || after.includes(skill)) {
+        continue;
+      }
+      (isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill);
+    }
+    const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp });
+    return [
+      ...before.map(skillCall),
+      { tool: "Bash", input: { command }, timestamp },
+      ...after.map(skillCall),
+    ];
+  }
+
+  if (item.type === "file_change") {
+    return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }];
+  }
+
+  const { id: _id, type, status: _status, ...input } = item;
+  return [{ tool: String(type), input, timestamp }];
+}
+
+// Codex reports commands as run through the user's login shell, e.g.
+// `/usr/bin/zsh -lc 'cat SKILL.md'`.
+export function unwrapShellCommand(command: string): string {
+  const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/);
+  if (!match) {
+    return command;
+  }
+  const body = match[2]!;
+  return match[1] === "'"
+    ? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'")
+    : body.replace(/\\(["\\$`])/g, "$1");
+}
+
+function isToolItem(item: CodexItem): boolean {
+  return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? "");
+}
+
+function parseEvent(line: string): CodexEvent | null {
+  try {
+    const value = JSON.parse(line);
+    return value && typeof value === "object" ? (value as CodexEvent) : null;
+  } catch {
+    return null;
+  }
+}

ai_evals/adapters/cli/mutation.ts

@@ -0,0 +1,10 @@
+export function isLikelyMutatingBashCommand(command: string): boolean {
+  return (
+    /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
+    /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
+    /\bsed\s+-i\b/.test(command) ||
+    /\bperl\s+-pi\b/.test(command) ||
+    // In command position only: `cat AGENTS.wmill.md` is a read.
+    /(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command)
+  );
+}

ai_evals/adapters/cli/runtime.test.ts

@@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test";
 import {
   anthropicUsageToBenchmarkTokenUsage,
   extractCliResultTokenUsage,
+  endpointEnv,
   extractProposedWmillCommands,
+  getFirstMutationToolIndex,
   parseWmillInvocationLog,
 } from "./runtime";
 
@@ -147,3 +149,32 @@ push
     ]);
   });
 });
+
+describe("getFirstMutationToolIndex", () => {
+  it("counts wmill only as a command, not in a file name", () => {
+    const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 });
+    expect(
+      getFirstMutationToolIndex([
+        bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"),
+        bash("ls && wmill generate-metadata"),
+      ])
+    ).toBe(1);
+    expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0);
+  });
+});
+
+describe("endpointEnv", () => {
+  it("passes the endpoint only its own key", () => {
+    const env = endpointEnv(
+      { baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" },
+      "m",
+      { PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" }
+    );
+    expect(env).toEqual({
+      PATH: "/bin",
+      ANTHROPIC_BASE_URL: "https://example.test",
+      ANTHROPIC_API_KEY: "ds",
+      ANTHROPIC_DEFAULT_HAIKU_MODEL: "m",
+    });
+  });
+});

ai_evals/adapters/cli/runtime.ts

@@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
 import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
 import { delimiter, join } from "path";
 import { fileURLToPath } from "url";
-import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models";
+import {
+  getCliEvalModel,
+  resolveEvalModel,
+  type CliEvalModelConfig,
+} from "../../core/models";
+import { runCodex, type AgentRunResult } from "./codex";
+import { isLikelyMutatingBashCommand } from "./mutation";
+export { formatCliRunModelLabel } from "../../core/models";
 import type {
   BenchmarkTokenUsage,
   CliToolInvocation,
@@ -141,16 +148,94 @@ export async function runPromptAndCapture(
   maxTurns: number = 3,
   modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
 ): Promise<PromptRunResult> {
-  const toolsUsed: ToolInvocation[] = [];
-  const skillsInvoked: string[] = [];
-  const bashCommands: string[] = [];
-  let output = "";
-  let assistantMessageCount = 0;
-  let tokenUsage: BenchmarkTokenUsage | null = null;
-  let finalContextTokens: number | null = null;
   const startedAt = Date.now();
   const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME);
   const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME);
+  const env = {
+    ...getQueryEnv(),
+    PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
+    WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
+  };
+
+  await installWmillStub(stubBinDir);
+
+  const run =
+    modelConfig.runtime === "codex"
+      ? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env)
+      : await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env);
+
+  const skillsInvoked: string[] = [];
+  const bashCommands: string[] = [];
+  for (const tool of run.toolsUsed) {
+    if (tool.tool === "Skill" && typeof tool.input.skill === "string") {
+      pushUnique(skillsInvoked, tool.input.skill);
+    }
+    if (tool.tool === "Bash") {
+      for (const command of extractBashCommands(tool.input)) {
+        pushUnique(bashCommands, command);
+      }
+    }
+  }
+
+  const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
+
+  return {
+    output: run.output,
+    durationMs: Date.now() - startedAt,
+    tokenUsage: run.tokenUsage,
+    finalContextTokens: run.finalContextTokens,
+    trace: {
+      toolsUsed: run.toolsUsed,
+      skillsInvoked,
+      assistantMessageCount: run.assistantMessageCount,
+      bashCommands,
+      proposedCommands: extractProposedWmillCommands(run.output),
+      executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
+      wmillInvocations,
+      firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed),
+    },
+  };
+}
+
+// Only the endpoint's own key may reach it: a missing key would make Claude Code
+// fall back to the user's Claude login, and the agent's Bash inherits this env,
+// so any other credential could end up in a tool result sent to the endpoint.
+export function endpointEnv(
+  endpoint: NonNullable<CliEvalModelConfig["anthropicEndpoint"]>,
+  model: string,
+  env: Record<string, string>
+): Record<string, string> {
+  const key = env[endpoint.apiKeyEnv];
+  if (!key) {
+    throw new Error(`${endpoint.apiKeyEnv} is not set`);
+  }
+  const rest = Object.fromEntries(
+    Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name))
+  );
+  return {
+    ...rest,
+    ANTHROPIC_BASE_URL: endpoint.baseUrl,
+    ANTHROPIC_API_KEY: key,
+    // Claude Code's background calls default to Haiku, which the endpoint does not serve.
+    ANTHROPIC_DEFAULT_HAIKU_MODEL: model,
+  };
+}
+
+async function runClaudeCode(
+  prompt: string,
+  cwd: string,
+  maxTurns: number,
+  modelConfig: CliEvalModelConfig,
+  env: Record<string, string>
+): Promise<AgentRunResult> {
+  const result: AgentRunResult = {
+    output: "",
+    toolsUsed: [],
+    assistantMessageCount: 0,
+    tokenUsage: null,
+    finalContextTokens: null,
+  };
+  const endpoint = modelConfig.anthropicEndpoint;
 
   const options: Options = {
     cwd,
@@ -158,80 +243,42 @@ export async function runPromptAndCapture(
     maxTurns,
     settingSources: ["project"],
     allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"],
-    env: {
-      ...getQueryEnv(),
-      PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
-      WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
-    },
+    env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env,
   };
 
-  await installWmillStub(stubBinDir);
-
   for await (const message of query({ prompt, options })) {
     if (message.type === "assistant") {
-      assistantMessageCount += 1;
+      result.assistantMessageCount += 1;
       const turnContext = anthropicUsageToBenchmarkTokenUsage(
         message.message?.usage
       )?.prompt;
       if (turnContext && turnContext > 0) {
-        finalContextTokens = turnContext;
+        result.finalContextTokens = turnContext;
       }
       const content = message.message?.content;
       if (Array.isArray(content)) {
         for (const block of content) {
           if (block.type === "tool_use") {
-            const input = normalizeToolInput(block.input);
-            toolsUsed.push({
+            result.toolsUsed.push({
               tool: block.name,
-              input,
+              input: normalizeToolInput(block.input),
               timestamp: Date.now()
             });
-
-            if (block.name === "Skill") {
-              const skillInput = input as { skill?: string };
-              if (skillInput.skill) {
-                pushUnique(skillsInvoked, skillInput.skill);
-              }
-            }
-
-            if (block.name === "Bash") {
-              for (const command of extractBashCommands(input)) {
-                pushUnique(bashCommands, command);
-              }
-            }
           } else if (block.type === "text") {
-            output += block.text;
+            result.output += block.text;
           }
         }
       }
     } else if (message.type === "result") {
       const resultMessage = message as { result?: string };
-      tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage;
+      result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage;
       if (typeof resultMessage.result === "string") {
-        output += resultMessage.result;
+        result.output += resultMessage.result;
       }
     }
   }
 
-  const proposedCommands = extractProposedWmillCommands(output);
-  const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
-
-  return {
-    output,
-    durationMs: Date.now() - startedAt,
-    tokenUsage,
-    finalContextTokens,
-    trace: {
-      toolsUsed,
-      skillsInvoked,
-      assistantMessageCount,
-      bashCommands,
-      proposedCommands,
-      executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
-      wmillInvocations,
-      firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed),
-    },
-  };
+  return result;
 }
 
 export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean {
@@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean
   return result.trace.toolsUsed.some((tool) => tool.tool === toolName);
 }
 
-export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string {
-  return `${modelConfig.provider}:${modelConfig.model}`;
-}
-
 export function getToolInputs(
   result: PromptRunResult,
   toolName: string
@@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string {
   return ["wmill", ...entry.argv].join(" ").trim();
 }
 
-function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
+export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
   for (const [index, tool] of toolsUsed.entries()) {
     if (tool.tool === "Write" || tool.tool === "Edit") {
       return index;
@@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
   return null;
 }
 
-function isLikelyMutatingBashCommand(command: string): boolean {
-  return (
-    /\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
-    /\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
-    /\bsed\s+-i\b/.test(command) ||
-    /\bperl\s+-pi\b/.test(command) ||
-    /\bwmill\b/.test(command)
-  );
-}
-
 function pushUnique(values: string[], value: string): void {
   if (!values.includes(value)) {
     values.push(value);

ai_evals/adapters/frontend/core/global/globalEvalRunner.ts

@@ -123,6 +123,7 @@ export interface GlobalEvalOptions {
   planMode?: boolean;
   model?: string;
   maxIterations?: number;
+  userAnswers?: string[];
   provider?: AIProvider;
   backend: WindmillBackendSettings;
   workspaceRoot?: string;
@@ -220,6 +221,7 @@ export async function runGlobalEval(
       onToolCall: options.runContext?.onToolCall,
       options: {
         maxIterations: options.maxIterations,
+        userAnswers: options.userAnswers,
         model,
         workspace: workspaceRoot,
         provider: options.provider,

ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts

@@ -85,6 +85,7 @@ export async function runEval<THelpers, TOutput>(
 
   const model = options.model ?? "gpt-4o";
   const maxIterations = options.maxIterations ?? 20;
+  const pendingAnswers = options.userAnswers && [...options.userAnswers];
   const workspace = options.workspace ?? "test-workspace";
   const provider = toFrontendEvalProvider(options.provider);
 
@@ -138,6 +139,12 @@ export async function runEval<THelpers, TOutput>(
     // edit the arguments, so a case can assert what the model proposed but never how
     // it reacts to the user changing something.
     requestRunArgs: async (_toolId, form) => form.args,
+    ...(pendingAnswers && {
+      requestUserQuestion: async () => {
+        const answer = pendingAnswers.shift();
+        return answer === undefined ? undefined : [answer];
+      },
+    }),
     onNewToken: (token: string) => {
       if (shouldEmitMessageStart) {
         onAssistantMessageStart?.();

ai_evals/adapters/frontend/core/shared/types.ts

@@ -16,6 +16,9 @@ export interface ToolCallDetail {
 export interface EvalRunnerOptions {
   backend: WindmillBackendSettings;
   maxIterations?: number;
+  // Scripted answers to the model's questions, one per question in order; a question past
+  // the last one is dismissed. Unset, the run cannot be asked anything.
+  userAnswers?: string[];
   model?: string;
   workspace?: string;
   provider?: AIProvider;

ai_evals/cases/global.yaml

@@ -198,22 +198,127 @@
     - uses the requested placeholder value and description
     - does not create a resource or deploy anything
 
+# A new app that has to store data, with nothing said about where: the data setup is the
+# user's call, so it is asked before anything is created. The question is dismissed here,
+# which leaves nothing the model may build on.
 - id: global-test7-ambiguous-app-asks-question
   prompt: |-
     Create a new raw app for triaging support tickets.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
   runtime:
-    maxTurns: 4
+    maxTurns: 6
+    userAnswers: []
   validate:
     draftCountExactly: 0
   toolExpect:
     requiredToolsUsed:
+      - list_datatables
       - askUserQuestion
     forbiddenToolsUsed:
       - init_app
+      - exec_datatable_sql
       - write_app_file
       - write_app_runnable
       - deploy_workspace_item
       - delete_workspace_item
+    toolCallArgs:
+      - tool: askUserQuestion
+        field: question
+        stringIncludesAnyOf:
+          - data table
+          - datatable
+          - schema
+          - tables
+          - store
+  skipJudge: true
+
+# The same request with the question answered: the app is built on what the user chose.
+- id: global-new-app-data-setup-answer-applied
+  prompt: |-
+    Create a new raw app for triaging support tickets.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 32
+    userAnswers:
+      - A new schema named helpdesk
+      - Create new tables
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: app
+        valueIncludes:
+          - helpdesk
+          - react-dom/client
+  toolExpect:
+    requiredToolsUsed:
+      - askUserQuestion
+      - init_app
+    forbiddenToolsUsed:
+      - deploy_workspace_item
+    toolCallArgs:
+      - tool: init_app
+        field: framework
+        stringEqualsAnyOf:
+          - react19
+      - tool: exec_datatable_sql
+        field: sql
+        stringIncludesAnyOf:
+          - helpdesk
+  skipJudge: true
+
+# No framework named, no app to follow, nothing to store: the modal's recommended
+# framework is used without a question.
+- id: global-new-app-defaults-to-react19
+  prompt: |-
+    Build me a little app that converts temperatures between Celsius and Fahrenheit.
+  runtime:
+    maxTurns: 12
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: app
+        valueIncludes:
+          - react-dom/client
+  toolExpect:
+    requiredToolsUsed:
+      - init_app
+    forbiddenToolsUsed:
+      - askUserQuestion
+      - exec_datatable_sql
+      - deploy_workspace_item
+    toolCallArgs:
+      - tool: init_app
+        field: framework
+        stringEqualsAnyOf:
+          - react19
+  skipJudge: true
+
+# The user said where the data goes, so there is nothing left to ask.
+- id: global-new-app-storage-specified-no-question
+  prompt: |-
+    Build a todo list app in React. Keep the todos in the main data table, in a new
+    schema called todo_app.
+  initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+  runtime:
+    maxTurns: 20
+  validate:
+    draftCountExactly: 1
+    requiredDrafts:
+      - type: app
+        valueIncludes:
+          - todo_app
+  toolExpect:
+    requiredToolsUsed:
+      - init_app
+      - write_app_runnable
+    forbiddenToolsUsed:
+      - askUserQuestion
+      - deploy_workspace_item
+    toolCallArgs:
+      - tool: exec_datatable_sql
+        field: sql
+        stringIncludesAnyOf:
+          - todo_app
   skipJudge: true
 
 - id: global-test8-human-script-infer-path-language

ai_evals/core/cases.test.ts

@@ -192,8 +192,6 @@ describe("loadCases", () => {
       requiredSkills: ["write-script-bun"],
       requiredSkillsBeforeFirstMutation: ["write-script-bun"],
       forbiddenSkills: ["write-script-python3", "write-flow"],
-      orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"],
-      orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
       forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"],
     });
   });

ai_evals/core/models.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "bun:test";
-import { resolveEvalModel } from "./models";
+import { formatRunModelLabel, resolveEvalModel } from "./models";
 
 describe("resolveEvalModel", () => {
   it("supports GPT-5.5 aliases for frontend evals", () => {
@@ -47,6 +47,18 @@ describe("resolveEvalModel", () => {
     });
   });
 
+  it("labels cli runs by agent, keeping Claude's historical label", () => {
+    expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe(
+      "anthropic:claude-sonnet-5-5",
+    );
+    expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe(
+      "codex:gpt-6.1-sol",
+    );
+    expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe(
+      "claude-code:deepseek-v4-pro",
+    );
+  });
+
   it("rejects Gemini aliases for cli evals", () => {
     expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow(
       "Model gemini-3-flash-preview is not supported for cli mode",

ai_evals/core/models.ts

@@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig {
   model: string;
 }
 
+// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts).
 export interface CliEvalModelConfig {
-  provider: "anthropic";
+  runtime: "claude-code" | "codex";
   model: string;
+  // An Anthropic-compatible endpoint other than Anthropic's, for claude-code,
+  // and the env var holding its key.
+  anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string };
 }
 
 export interface EvalModelSpec {
@@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "claude-haiku-4-5-20251001",
     },
     cli: {
-      provider: "anthropic",
+      runtime: "claude-code",
       model: "haiku",
     },
   },
@@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "claude-sonnet-4-5-20250929",
     },
     cli: {
-      provider: "anthropic",
+      runtime: "claude-code",
       model: "sonnet",
     },
   },
@@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "claude-opus-4-6",
     },
     cli: {
-      provider: "anthropic",
+      runtime: "claude-code",
       model: "opus",
     },
   },
@@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "claude-sonnet-5-5",
     },
     cli: {
-      provider: "anthropic",
+      runtime: "claude-code",
       model: "claude-sonnet-5-5",
     },
   },
@@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "claude-opus-5-5",
     },
     cli: {
-      provider: "anthropic",
+      runtime: "claude-code",
       model: "claude-opus-5-5",
     },
   },
@@ -139,6 +143,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       provider: "openai",
       model: "gpt-6-astra",
     },
+    cli: {
+      runtime: "codex",
+      model: "gpt-6-astra",
+    },
+  },
+  {
+    id: "gpt-6.1-sol",
+    label: "GPT-6.1 Sol",
+    aliases: ["gpt-6.1-sol"],
+    frontend: {
+      provider: "openai",
+      model: "gpt-6.1-sol",
+    },
   },
   {
     id: "gpt-6-sol",
@@ -149,6 +166,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       model: "gpt-6-sol",
     },
   },
+  {
+    id: "gpt-6.1-sol",
+    label: "GPT-6.1 Sol",
+    aliases: ["gpt-6.1-sol"],
+    frontend: {
+      provider: "openai",
+      model: "gpt-6.1-sol",
+    },
+    cli: {
+      runtime: "codex",
+      model: "gpt-6.1-sol",
+    },
+  },
   {
     id: "gpt-6-luna",
     label: "GPT-6 Luna",
@@ -215,6 +245,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [
       provider: "deepseek",
       model: "deepseek-v4-pro",
     },
+    cli: {
+      runtime: "claude-code",
+      model: "deepseek-v4-pro",
+      anthropicEndpoint: {
+        baseUrl: "https://api.deepseek.com/anthropic",
+        apiKeyEnv: "DEEPSEEK_API_KEY",
+      },
+    },
   },
 ];
 
@@ -253,11 +291,20 @@ export function formatRunModelLabel(
   model: EvalModelSpec,
 ): string {
   if (mode === "cli") {
-    return `${model.cli!.provider}:${model.cli!.model}`;
+    return formatCliRunModelLabel(model.cli!);
   }
   return `${model.frontend!.provider}:${model.frontend!.model}`;
 }
 
+// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was
+// recorded under; any other pairing names the agent.
+export function formatCliRunModelLabel(config: CliEvalModelConfig): string {
+  if (config.runtime === "claude-code" && !config.anthropicEndpoint) {
+    return `anthropic:${config.model}`;
+  }
+  return `${config.runtime}:${config.model}`;
+}
+
 export function getFrontendEvalModel(
   model: EvalModelSpec,
 ): FrontendEvalModelConfig {

ai_evals/core/types.ts

@@ -29,6 +29,10 @@ export interface EvalCaseRuntimeAppContextSpec {
 
 export interface EvalCaseRuntimeSpec {
   maxTurns?: number;
+  // Global mode: scripted answers to askUserQuestion, one per question in order; a question
+  // past the last one is dismissed, so `[]` dismisses them all. Unset, the question tool
+  // reports that it cannot ask.
+  userAnswers?: string[];
   backendPreview?: EvalCaseRuntimeBackendPreview;
   appContext?: EvalCaseRuntimeAppContextSpec;
   // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat.

ai_evals/modes/cli.ts

@@ -39,17 +39,36 @@ interface CliRunActual {
   trace: CliTrace;
 }
 
-const CLAUDE_PROJECT_PREAMBLE = [
-  "Follow the project instructions from AGENTS.md exactly.",
-  "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
-  "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
-  "Do not skip the Skill step.",
+// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex
+// loads a skill by reading its SKILL.md.
+const SKILL_STEP: Record<CliEvalModelConfig["runtime"], string[]> = {
+  "claude-code": [
+    "Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
+    "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
+    "Do not skip the Skill step.",
+  ],
+  codex: [
+    "Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.",
+    "Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
+    "Do not skip the skill step.",
+  ],
+};
+
+const HARNESS_PREAMBLE = [
   "You are running inside an automated benchmark harness, not an interactive user session.",
   "Act autonomously and complete the requested file changes directly in the workspace.",
   "Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.",
   "Do not respond with a plan when you can make the change directly.",
   "Only describe what was done after you have written the files.",
-].join(" ");
+];
+
+function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string {
+  return [
+    "Follow the project instructions from AGENTS.md exactly.",
+    ...SKILL_STEP[runtime],
+    ...HARNESS_PREAMBLE,
+  ].join(" ");
+}
 
 export function createCliModeRunner(
   modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
@@ -89,7 +108,7 @@ export function createCliModeRunner(
         });
         await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
 
-        const renderedPrompt = await renderPrompt(prompt, workspaceDir);
+        const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime);
         const run = await runPromptAndCapture(
           renderedPrompt,
           workspaceDir,
@@ -193,7 +212,11 @@ export function getCliRunModelLabel(
   return formatCliRunModelLabel(modelConfig);
 }
 
-async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
+async function renderPrompt(
+  prompt: string,
+  workspaceDir: string,
+  runtime: CliEvalModelConfig["runtime"]
+): Promise<string> {
   const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
   const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8");
 
@@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise<strin
     agentsInstructions.trim(),
     "",
     "# Benchmark Harness",
-    CLAUDE_PROJECT_PREAMBLE,
+    projectPreamble(runtime),
     "",
     "# User Request",
     renderedUserPrompt,

ai_evals/modes/global.ts

@@ -54,6 +54,7 @@ export function createGlobalModeRunner(
           sessionChat: context.evalCase?.runtime?.sessionChat,
           planMode: context.evalCase?.runtime?.planMode,
           maxIterations: context.evalCase?.runtime?.maxTurns,
+          userAnswers: context.evalCase?.runtime?.userAnswers,
           provider: modelConfig.provider,
           model: modelConfig.model,
           backend: backendSettings,

backend/.sqlx/query-03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\nINSERT INTO concurrency_counter(concurrency_id, job_uuids)\nVALUES ($1, '{}'::jsonb)\nON CONFLICT (concurrency_id)\nDO UPDATE SET job_uuids = concurrency_counter.job_uuids\nRETURNING (SELECT COUNT(*)\n           FROM jsonb_object_keys(job_uuids) AS keys(key)\n           WHERE key <> $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a"
+}

backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n            WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "draining",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b"
+}

backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json

@@ -1,11 +1,11 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)",
+  "query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running",
   "describe": {
     "columns": [
       {
         "ordinal": 0,
-        "name": "exists",
+        "name": "?column?",
         "type_info": "Bool"
       }
     ],
@@ -19,5 +19,5 @@
       null
     ]
   },
-  "hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e"
+  "hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0"
 }

backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT wp.draining FROM v2_job j\n            JOIN v2_job_queue q ON q.id = j.id\n            JOIN worker_ping wp ON wp.worker = q.worker\n            WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n            AND ($3::text[] IS NULL OR j.tag = ANY($3))",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "draining",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13"
+}

backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "coalesce",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683"
+}

backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "coalesce",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c"
+}

backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json

@@ -1,33 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n        DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n        RETURNING jobs_executed",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "jobs_executed",
-        "type_info": "Int4"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Varchar",
-        "Varchar",
-        "Text",
-        "TextArray",
-        "Varchar",
-        "Varchar",
-        "TextArray",
-        "Varchar",
-        "Int8",
-        "Int8",
-        "Text",
-        "Bool"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc"
-}

backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT q.id FROM v2_job_queue q\n            JOIN native_retry_attempt n ON n.job_id = q.id\n            JOIN v2_job r ON r.id = q.id\n            JOIN v2_job a ON a.id = r.parent_job\n            WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429"
+}

backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE worker_ping SET draining = true WHERE worker = $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919"
+}

backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json

@@ -0,0 +1,34 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n        DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n        RETURNING jobs_executed",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "jobs_executed",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Text",
+        "TextArray",
+        "Varchar",
+        "Varchar",
+        "TextArray",
+        "Varchar",
+        "Int8",
+        "Int8",
+        "Text",
+        "Bool",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b"
+}

backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n            SELECT c.id, c.workspace_id, NULL, NULL, true\n                FROM v2_job_completed c\n                JOIN v2_job j ON j.id = c.id\n                WHERE c.status = 'failure'\n                    AND c.workspace_id = $2\n                    AND j.flow_step_id IS NULL\n                    AND j.runnable_id IS NOT DISTINCT FROM $3\n                    AND EXISTS (\n                        SELECT 1 FROM v2_job_completed sc\n                        JOIN v2_job sj ON sj.id = sc.id\n                        JOIN native_retry_attempt nra ON nra.job_id = sc.id\n                        WHERE sc.status = 'success'\n                            AND sc.workspace_id = $2\n                            AND sj.parent_job = $1\n                            AND sj.runnable_id IS NOT DISTINCT FROM $3\n                    )\n                    AND (\n                        (j.parent_job = $1\n                            AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n                        OR (c.id = $1 AND j.parent_job IS NULL\n                            AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n                    )\n            ON CONFLICT (job_id) DO NOTHING",
+  "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n            SELECT c.id, c.workspace_id, NULL, NULL, true\n                FROM v2_job_completed c\n                JOIN v2_job j ON j.id = c.id\n                WHERE c.status = 'failure'\n                    AND c.workspace_id = $2\n                    AND j.flow_step_id IS NULL\n                    AND j.runnable_id IS NOT DISTINCT FROM $3\n                    AND EXISTS (\n                        SELECT 1 FROM v2_job_completed sc\n                        JOIN v2_job sj ON sj.id = sc.id\n                        JOIN native_retry_attempt nra ON nra.job_id = sc.id\n                        WHERE sc.status = 'success'\n                            AND sc.workspace_id = $2\n                            AND sj.parent_job = $1\n                            AND sj.runnable_id IS NOT DISTINCT FROM $3\n                    )\n                    AND (\n                        (j.parent_job = $1\n                            AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n                        OR (c.id = $1\n                            AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n                    )\n            ON CONFLICT (job_id) DO NOTHING",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258"
+  "hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9"
 }

backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n        CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n        FROM worker_ping\n        WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n        ORDER BY ping_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -107,6 +107,11 @@
         "ordinal": 20,
         "name": "native_mode",
         "type_info": "Bool"
+      },
+      {
+        "ordinal": 21,
+        "name": "draining",
+        "type_info": "Bool"
       }
     ],
     "parameters": {
@@ -138,8 +143,9 @@
       true,
       true,
       true,
+      false,
       false
     ]
   },
-  "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca"
+  "hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c"
 }

backend/Cargo.lock

@@ -6685,9 +6685,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "inventory"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
+checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c"
 dependencies = [
  "rustversion",
 ]
@@ -9617,7 +9617,7 @@ version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
- "toml_edit 0.25.15+spec-1.1.0",
+ "toml_edit 0.25.16+spec-1.1.0",
 ]
 
 [[package]]
@@ -13622,9 +13622,9 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "1.1.1+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053"
 dependencies = [
  "serde_core",
 ]
@@ -13644,21 +13644,21 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.25.15+spec-1.1.0"
+version = "0.25.16+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"
+checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722"
 dependencies = [
  "indexmap 2.14.2",
- "toml_datetime 1.1.1+spec-1.1.0",
+ "toml_datetime 1.1.2+spec-1.1.0",
  "toml_parser",
  "winnow 1.0.4",
 ]
 
 [[package]]
 name = "toml_parser"
-version = "1.1.3+spec-1.1.0"
+version = "1.1.4+spec-1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
+checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333"
 dependencies = [
  "winnow 1.0.4",
 ]
@@ -14829,7 +14829,7 @@ dependencies = [
 
 [[package]]
 name = "windmill"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -14918,7 +14918,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-ai"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -14952,7 +14952,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-alerting"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -14965,7 +14965,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "argon2",
@@ -15105,7 +15105,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-agent-workers"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15128,7 +15128,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-assets"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15145,7 +15145,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-auth"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15171,7 +15171,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-client"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "reqwest 0.12.28",
  "serde",
@@ -15181,7 +15181,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-configs"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15198,7 +15198,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-debug"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "base64 0.22.1",
@@ -15220,7 +15220,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-embeddings"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15243,7 +15243,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flow-conversations"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15259,7 +15259,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-flows"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15281,7 +15281,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-groups"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15303,7 +15303,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-inputs"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15317,7 +15317,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-integration-tests"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -15353,7 +15353,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-jobs"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15372,13 +15372,14 @@ dependencies = [
  "uuid",
  "windmill-api-auth",
  "windmill-api-sse",
+ "windmill-audit",
  "windmill-common",
  "windmill-queue",
 ]
 
 [[package]]
 name = "windmill-api-npm-proxy"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15406,7 +15407,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-openapi"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15428,7 +15429,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-schedule"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15448,7 +15449,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-scripts"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15490,7 +15491,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-settings"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15519,7 +15520,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-sse"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "lazy_static",
  "serde",
@@ -15531,7 +15532,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-users"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "argon2",
  "axum 0.8.9",
@@ -15555,7 +15556,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workers"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15569,7 +15570,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-api-workspaces"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "axum 0.8.9",
  "chrono",
@@ -15608,7 +15609,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-audit"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "chrono",
  "lazy_static",
@@ -15617,12 +15618,13 @@ dependencies = [
  "sql-builder",
  "sqlx",
  "tracing",
+ "tracing-subscriber",
  "windmill-common",
 ]
 
 [[package]]
 name = "windmill-autoscaling"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "axum 0.8.9",
@@ -15641,7 +15643,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-common"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "aes-gcm",
  "aho-corasick",
@@ -15755,7 +15757,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-dep-map"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "chrono",
  "futures",
@@ -15775,7 +15777,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-git-sync"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "regex",
  "serde",
@@ -15792,7 +15794,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-indexer"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -15819,7 +15821,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-jseval"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -15836,7 +15838,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "itertools 0.14.0",
  "lazy_static",
@@ -15852,7 +15854,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-mcp"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -15873,7 +15875,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-native-triggers"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -15904,7 +15906,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-oauth"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "arc-swap",
@@ -15929,7 +15931,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-object-store"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-stream",
@@ -15965,7 +15967,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-operator"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "futures",
@@ -15983,7 +15985,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "convert_case 0.6.0",
  "serde",
@@ -15992,7 +15994,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16004,7 +16006,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -16016,7 +16018,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -16028,7 +16030,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16040,7 +16042,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -16052,7 +16054,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -16063,7 +16065,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -16074,7 +16076,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -16086,7 +16088,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -16097,7 +16099,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -16119,7 +16121,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -16131,7 +16133,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16145,7 +16147,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "convert_case 0.6.0",
@@ -16162,7 +16164,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16175,7 +16177,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -16187,7 +16189,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16205,7 +16207,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -16221,7 +16223,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -16237,7 +16239,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -16251,7 +16253,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-queue"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -16290,7 +16292,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-runtime-nativets"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "const_format",
@@ -16330,7 +16332,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-sql-datatype-parser-wasm"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "getrandom 0.3.4",
  "wasm-bindgen",
@@ -16341,7 +16343,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-store"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -16376,7 +16378,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-test-utils"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16400,7 +16402,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16433,7 +16435,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-amqp"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16460,7 +16462,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-azure"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16493,7 +16495,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-email"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16513,7 +16515,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-gcp"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16547,7 +16549,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-http"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16584,7 +16586,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-kafka"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16607,7 +16609,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-mqtt"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16631,7 +16633,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-nats"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-nats",
@@ -16655,7 +16657,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-postgres"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16690,7 +16692,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-sqs"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16718,7 +16720,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-trigger-websocket"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -16743,7 +16745,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "bitflags 2.13.2",
@@ -16762,7 +16764,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-once-cell",
@@ -16881,7 +16883,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-worker-volumes"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "bytes",
  "futures",

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.826.0"
+version = "1.828.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -88,7 +88,7 @@ members = [
 exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
 
 [workspace.package]
-version = "1.826.0"
+version = "1.828.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-065c2c64c3904d5c0b2766b424210d34f9e54d2d
+a37413ccdd52e65bc240bad3e87084bf4100e119

backend/migrations/20261008070335_worker_ping_draining.down.sql

@@ -0,0 +1 @@
+ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining;

backend/migrations/20261008070335_worker_ping_draining.up.sql

@@ -0,0 +1 @@
+ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE;

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -1040,11 +1040,13 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ {
 
 pub fn parse_snowflake_typ(typ: &str) -> Typ {
     match typ {
-        "varchar" => Typ::Str(None),
-        "binary" => Typ::Bytes,
-        "date" | "time" | "timestamp" => Typ::Datetime,
-        "int" => Typ::Int,
-        "float" => Typ::Float,
+        "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None),
+        "binary" | "varbinary" => Typ::Bytes,
+        "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz"
+        | "timestamp_tz" => Typ::Datetime,
+        "int" | "integer" | "bigint" | "smallint" | "tinyint" | "byteint" => Typ::Int,
+        "float" | "float4" | "float8" | "double" | "double precision" | "real" | "number"
+        | "decimal" | "numeric" => Typ::Float,
         "boolean" => Typ::Bool,
         "s3object" => Typ::Resource("S3Object".to_string()),
         _ => Typ::Str(None),

backend/parsers/windmill-parser-wasm/Cargo.lock

@@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windmill-common"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "aho-corasick",
  "anyhow",
@@ -6302,7 +6302,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-macros"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6314,7 +6314,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "convert_case",
  "serde",
@@ -6323,7 +6323,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-bash"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6335,7 +6335,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-csharp"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6347,7 +6347,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-go"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "gosyn",
@@ -6359,7 +6359,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-graphql"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6371,7 +6371,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-java"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6383,7 +6383,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-nu"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "nu-parser",
@@ -6394,7 +6394,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-php"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6405,7 +6405,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "itertools 0.14.0",
@@ -6417,7 +6417,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6428,7 +6428,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-py-imports"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "async-recursion",
@@ -6450,7 +6450,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-r"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde_json",
@@ -6462,7 +6462,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ruby"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-rust"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "convert_case",
@@ -6493,7 +6493,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-sql-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde",
@@ -6518,7 +6518,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6536,7 +6536,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-ts-asset"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "serde-wasm-bindgen",
@@ -6552,7 +6552,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wac"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "rustpython-ast",
@@ -6568,7 +6568,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-wasm"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "getrandom 0.2.17",
@@ -6600,7 +6600,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-parser-yaml"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "lazy_static",
@@ -6614,7 +6614,7 @@ dependencies = [
 
 [[package]]
 name = "windmill-types"
-version = "1.826.0"
+version = "1.828.0"
 dependencies = [
  "anyhow",
  "bitflags",

backend/parsers/windmill-parser-wasm/Cargo.toml

@@ -12,7 +12,7 @@ resolver = "2"
 members = ["."]
 
 [workspace.package]
-version = "1.826.0"
+version = "1.828.0"
 edition = "2021"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 

backend/src/monitor.rs

@@ -40,7 +40,7 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts};
 use windmill_common::global_settings::OAUTH_SETTING;
 use windmill_common::otel_oss::{
     otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool,
-    otel_set_queue_count, otel_set_queue_running_count,
+    otel_set_queue_count, otel_set_queue_delay, otel_set_queue_running_count,
 };
 use windmill_common::{
     agent_workers::DECODED_AGENT_TOKEN,
@@ -168,6 +168,12 @@ lazy_static::lazy_static! {
         &["tag"]
     ).unwrap();
 
+    static ref QUEUE_DELAY_SECONDS: prometheus::GaugeVec = prometheus::register_gauge_vec!(
+        "queue_delay_seconds",
+        "How long the next job to be picked up has been waiting, in seconds",
+        &["tag"]
+    ).unwrap();
+
     static ref QUEUE_RUNNING_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
         "queue_running_count",
         "Number of running jobs in the queue",
@@ -421,6 +427,8 @@ pub async fn initial_load(
                     native_mode,
                     // an agent worker never reads its group's config, only its token
                     object_store_cache_config: None,
+                    // nor whether it is paused: the server refuses it jobs instead
+                    paused: false,
                 }));
             }
         }
@@ -5597,6 +5605,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
         for q in QUEUE_COUNT_TAGS.read().await.iter() {
             if queue_stats.get(q).is_none() {
                 (*QUEUE_COUNT).with_label_values(&[q]).set(0);
+                (*QUEUE_DELAY_SECONDS).with_label_values(&[q]).set(0.0);
             }
         }
     }
@@ -5605,6 +5614,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
         for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() {
             if queue_stats.get(q).is_none() {
                 otel_set_queue_count(q, 0);
+                otel_set_queue_delay(q, 0.0);
             }
         }
     }
@@ -5620,6 +5630,9 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
         if metrics_enabled {
             let metric = (*QUEUE_COUNT).with_label_values(&[tag]);
             metric.set(count as i64);
+            (*QUEUE_DELAY_SECONDS)
+                .with_label_values(&[tag])
+                .set(stat.delay);
             tags_to_watch.push(tag.to_string());
         }
 
@@ -5627,6 +5640,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
             otel_tags_to_watch.push(tag.to_string());
         }
         otel_set_queue_count(tag, count as i64);
+        otel_set_queue_delay(tag, stat.delay);
     }
 
     if save_metrics {
@@ -5677,8 +5691,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b
                 .as_ref()
                 .is_some_and(|dws| !dws.is_empty());
 
+        // Pausing or resuming must not restart the workers, dedicated ones included: a reload
+        // whose only difference is the flag skips every restart below.
+        let only_pause_changed = wc.paused != config.paused
+            && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc;
+
         if **wc != config || has_dedicated {
-            if kill_if_change {
+            if kill_if_change && !only_pause_changed {
                 if has_dedicated
                     || wc.dedicated_worker != config.dedicated_worker
                     || wc.dedicated_workers != config.dedicated_workers

backend/summarized_schema.txt

@@ -219,7 +219,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
 windmill_migrations: name(text), created_at(ts)
 worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
   FK: (workspace_id) -> workspace(id)
-worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
+worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool)
 workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool)
   FK: (parent_workspace_id) -> workspace(id)
 workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)

backend/tests/ai_decision.rs

@@ -1,5 +1,6 @@
-//! AI decision steps run end to end against a stand-in for System One endpoints: TypeSafe's, and
-//! Cloudflare Workers AI's, which serves each model at its own URL inside an API envelope.
+//! AI decision steps run end to end against a stand-in for the decision endpoints: TypeSafe's
+//! System One, Cloudflare Workers AI's, which serves each model at its own URL inside an API
+//! envelope, and OpenAI's Decisions API.
 
 use axum::{routing::post, Json, Router};
 use serde_json::{json, Value};
@@ -37,6 +38,19 @@ async fn start_decision_stub() -> anyhow::Result<u16> {
             post(|Json(body): Json<Value>| async move {
                 Json(json!({"result": refund_answers(&body), "success": true, "errors": []}))
             }),
+        )
+        // OpenAI's Decisions API takes the questions as a list and answers with one.
+        .route(
+            "/openai/v1/decisions",
+            post(|Json(body): Json<Value>| async move {
+                let answers = body["questions"]
+                    .as_array()
+                    .into_iter()
+                    .flatten()
+                    .map(|q| json!({"type": "choice", "name": q["name"], "choice": "refund"}))
+                    .collect::<Vec<_>>();
+                Json(json!({"answers": answers}))
+            }),
         );
     let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
     let port = listener.local_addr()?.port();
@@ -72,6 +86,11 @@ async fn test_ai_decision_answers_route_a_following_branch(
             },
             "model": "clef-flash"
         }),
+        json!({
+            "kind": "openai",
+            "resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/openai/v1")},
+            "model": "gpt-6-luna"
+        }),
     ] {
         let flow: FlowValue = serde_json::from_value(json!({
             "modules": [

backend/tests/error_handler.rs

@@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
 
     Ok(())
 }
+
+/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
+/// and its failure must reach the workspace error handler like a workspace script's.
+#[cfg(all(feature = "enterprise", feature = "private"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_error_handler_triggered_on_hub_script_failure(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    use windmill_common::jobs::JobPayload;
+
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+
+    let version = "990000002";
+    let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
+    tokio::fs::create_dir_all(hub_dir).await?;
+    tokio::fs::write(
+        format!("{hub_dir}/{version}"),
+        r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
+    )
+    .await?;
+
+    // The workspace handler settings are cached per workspace id across the tests of this
+    // binary, so this reuses the handler path every other test here configures.
+    sqlx::query(
+        "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
+         VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
+    )
+    .execute(&db)
+    .await?;
+    sqlx::query(
+        r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb
+           WHERE workspace_id = 'test-workspace'"#,
+    )
+    .execute(&db)
+    .await?;
+
+    let completed_job = RunJob::from(JobPayload::ScriptHub {
+        path: format!("hub/{version}/test/fail"),
+        apply_preprocessor: false,
+    })
+    .run_until_complete(&db, false, server.addr.port())
+    .await;
+    assert!(!completed_job.success, "Hub script should have failed");
+
+    let mut handler_parent = None;
+    for _ in 0..50 {
+        handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
+            "SELECT parent_job FROM v2_job
+             WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'",
+        )
+        .fetch_optional(&db)
+        .await?;
+        if handler_parent.is_some() {
+            break;
+        }
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+    }
+    assert_eq!(
+        handler_parent,
+        Some(Some(completed_job.id)),
+        "Workspace error handler should handle the failed hub script"
+    );
+
+    Ok(())
+}

backend/tests/job_token_scopes.rs

@@ -410,5 +410,22 @@ async fn test_oidc_claim_carries_the_presented_scopes(db: Pool<Postgres>) -> any
         )?;
         assert_eq!(claims["job_token_scopes"], expected, "{job}");
     }
+
+    // A soft-canceled job is still `running` until its worker kills it.
+    sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1")
+        .bind(Uuid::parse_str(OIDC_JOB)?)
+        .execute(&db)
+        .await?;
+    let resp = client
+        .post(format!("{base}/oidc/token/test-audience"))
+        .bearer_auth(job_token(&db, OIDC_JOB).await?)
+        .send()
+        .await?;
+    assert_eq!(
+        resp.status(),
+        StatusCode::FORBIDDEN,
+        "{}",
+        resp.text().await?
+    );
     Ok(())
 }

backend/tests/otel.rs

@@ -168,6 +168,7 @@ async fn test_all_metrics_e2e() {
     // ── Gauges ──────────────────────────────────────────────────────
 
     otel_set_queue_count("python3", 42);
+    otel_set_queue_delay("python3", 12.5);
     otel_set_queue_running_count("deno", 5);
     otel_set_worker_busy("worker-test-1", 1);
     otel_set_db_pool(5, 10, 20);
@@ -188,7 +189,7 @@ async fn test_all_metrics_e2e() {
     let metrics = flush_and_get_metrics(&state);
     let names = metric_names(&metrics);
 
-    // ── Verify all 20 metric names are present ──────────────────────
+    // ── Verify all 21 metric names are present ──────────────────────
 
     let expected = [
         "windmill.queue.push_count",
@@ -197,6 +198,7 @@ async fn test_all_metrics_e2e() {
         "windmill.queue.zombie_restart_count",
         "windmill.queue.zombie_delete_count",
         "windmill.queue.count",
+        "windmill.queue.delay",
         "windmill.queue.running_count",
         "windmill.worker.execution_count",
         "windmill.worker.execution_duration",
@@ -261,6 +263,9 @@ async fn test_all_metrics_e2e() {
         .expect("queue.count data point with tag=python3 not found");
     assert_eq!(dp.1, 42);
 
+    let m = find_metric(&metrics, "windmill.queue.delay").unwrap();
+    assert!((gauge_f64_value(m).unwrap() - 12.5).abs() < f64::EPSILON);
+
     let m = find_metric(&metrics, "windmill.queue.running_count").unwrap();
     let values = gauge_i64_values(m);
     let dp = values

backend/tests/success_handler.rs

@@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow:
 
     Ok(())
 }
+
+/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
+/// and its success must reach the workspace success handler like a workspace script's.
+#[cfg(all(feature = "enterprise", feature = "private"))]
+#[sqlx::test(fixtures("base"))]
+async fn test_success_handler_triggered_on_hub_script_success(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    use windmill_common::jobs::JobPayload;
+
+    initialize_tracing().await;
+
+    let server = ApiServer::start(db.clone()).await?;
+
+    let version = "990000003";
+    let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
+    tokio::fs::create_dir_all(hub_dir).await?;
+    tokio::fs::write(
+        format!("{hub_dir}/{version}"),
+        r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
+    )
+    .await?;
+
+    // The workspace handler settings are cached per workspace id across the tests of this
+    // binary, so this reuses the handler path every other test here configures.
+    sqlx::query(
+        "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
+         VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
+    )
+    .execute(&db)
+    .await?;
+    sqlx::query(
+        r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb
+           WHERE workspace_id = 'test-workspace'"#,
+    )
+    .execute(&db)
+    .await?;
+
+    let completed_job = RunJob::from(JobPayload::ScriptHub {
+        path: format!("hub/{version}/test/ok"),
+        apply_preprocessor: false,
+    })
+    .run_until_complete(&db, false, server.addr.port())
+    .await;
+    assert!(completed_job.success, "Hub script should have succeeded");
+
+    let mut handler_parent = None;
+    for _ in 0..50 {
+        handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
+            "SELECT parent_job FROM v2_job
+             WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'",
+        )
+        .fetch_optional(&db)
+        .await?;
+        if handler_parent.is_some() {
+            break;
+        }
+        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
+    }
+    assert_eq!(
+        handler_parent,
+        Some(Some(completed_job.id)),
+        "Workspace success handler should handle the successful hub script"
+    );
+
+    Ok(())
+}

backend/tests/worker.rs

@@ -4698,6 +4698,129 @@ async fn test_run_wait_result_early_return_with_failure_module(
     Ok(())
 }
 
+/// `recover: true` from the error handler of a step that fails inside a loop turns the flow
+/// green, as it does for a top-level step, without changing which steps run: a loop that
+/// stops at a failed iteration still stops there, one that skips failures still carries on.
+#[cfg(feature = "deno_core")]
+#[sqlx::test(fixtures("base"))]
+async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let failing_loop = |skip_failures: bool| {
+        json!({
+            "id": "loop",
+            "value": {
+                "type": "forloopflow",
+                "iterator": { "type": "static", "value": [1, 2, 3] },
+                "skip_failures": skip_failures,
+                "modules": [{
+                    "id": "a",
+                    "value": {
+                        "input_transforms": {},
+                        "type": "rawscript",
+                        "language": "deno",
+                        "content": "export function main() { throw new Error('boom'); }",
+                    },
+                }],
+            },
+        })
+    };
+    let run = |loop_step: serde_json::Value| {
+        let flow: FlowValue = serde_json::from_value(json!({
+            "modules": [loop_step, {
+                "id": "b",
+                "value": {
+                    "input_transforms": {},
+                    "type": "rawscript",
+                    "language": "deno",
+                    "content": "export function main() { return { ran_b: true } }",
+                },
+            }],
+            "failure_module": {
+                "value": {
+                    "input_transforms": {},
+                    "type": "rawscript",
+                    "language": "deno",
+                    "content": "export function main() { return { handled: true, recover: true } }",
+                },
+            },
+        }))
+        .unwrap();
+        RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
+            .run_until_complete(&db, false, port)
+    };
+    let handler_runs = || {
+        sqlx::query_scalar::<_, i64>(
+            "SELECT count(*) FROM v2_job_completed c JOIN v2_job j USING (id)
+             WHERE j.kind = 'preview' AND c.result @> '{\"handled\": true}'::jsonb",
+        )
+        .fetch_one(&db)
+    };
+
+    let stopped = run(failing_loop(false)).await;
+    let stopped_handler_runs = handler_runs().await?;
+    let skipping = run(failing_loop(true)).await;
+    let skipping_handler_runs = handler_runs().await? - stopped_handler_runs;
+    // A parallel node counts its iterations' stored status: an iteration that ended on a
+    // recovered failure must still be stored as failed there.
+    let parallel = run(json!({
+        "id": "par",
+        "value": {
+            "type": "forloopflow",
+            "iterator": { "type": "static", "value": [1, 2] },
+            "parallel": true,
+            "skip_failures": false,
+            "modules": [failing_loop(false)],
+        },
+    }))
+    .await;
+    // A recovery the inner loop absorbs, then an error raised starting the next step: that
+    // error is a new failure and must leave the flow failed.
+    #[cfg(feature = "quickjs")]
+    let chaining_error = run(json!({
+        "id": "outer",
+        "value": {
+            "type": "forloopflow",
+            "iterator": { "type": "static", "value": [1] },
+            "skip_failures": false,
+            "modules": [failing_loop(true), {
+                "id": "guarded",
+                "skip_if": { "expr": "missingFunction()" },
+                "value": { "input_transforms": {}, "type": "identity" },
+            }],
+        },
+    }))
+    .await;
+
+    server.close().await.unwrap();
+
+    #[cfg(feature = "quickjs")]
+    assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed");
+
+    assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success");
+    assert!(
+        stopped.json_result().unwrap().get("ran_b").is_none(),
+        "the step after the loop must not run, as without recovery"
+    );
+    assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration");
+
+    assert!(skipping.success);
+    assert_eq!(
+        skipping.json_result().unwrap(),
+        json!({ "ran_b": true }),
+        "a loop that skips failures runs every iteration and the following step"
+    );
+    assert_eq!(skipping_handler_runs, 3);
+
+    assert!(
+        !parallel.json_result().unwrap().to_string().contains("ran_b"),
+        "the parallel loop fails as without recovery, so the step after it must not run"
+    );
+    Ok(())
+}
+
 #[cfg(feature = "python")]
 #[sqlx::test(fixtures("base"))]
 async fn test_flow_lock_all(db: Pool<Postgres>) -> anyhow::Result<()> {

backend/tests/worker_group_pause.rs

@@ -0,0 +1,71 @@
+//! In its own test binary: it flips the process-wide `WORKER_CONFIG`, which would stall every
+//! other worker test sharing the process.
+
+use futures::StreamExt;
+use sqlx::{Pool, Postgres};
+use std::time::Duration;
+use windmill_common::{
+    jobs::{JobPayload, RawCode},
+    scripts::ScriptLang,
+    worker::{Connection, WORKER_CONFIG},
+};
+use windmill_test_utils::*;
+
+fn set_paused(paused: bool) {
+    let mut wc = (**WORKER_CONFIG.load()).clone();
+    wc.paused = paused;
+    WORKER_CONFIG.store(std::sync::Arc::new(wc));
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn paused_worker_leaves_jobs_queued_until_resumed(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    set_paused(true);
+    let id = RunJob::from(JobPayload::Code(RawCode {
+        hash: None,
+        content: "echo done".to_string(),
+        path: None,
+        lock: None,
+        language: ScriptLang::Bash,
+        cache_ttl: None,
+        cache_ignore_s3_path: None,
+        dedicated_worker: None,
+        concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
+            .into(),
+        debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
+        modules: None,
+        tag: None,
+    }))
+    .push(&db)
+    .await;
+
+    let mut completed = listen_for_completed_jobs(&db).await;
+    in_test_worker(
+        Connection::Sql(db.clone()),
+        async {
+            tokio::time::sleep(Duration::from_secs(3)).await;
+            let running =
+                sqlx::query_scalar::<_, bool>("SELECT running FROM v2_job_queue WHERE id = $1")
+                    .bind(id)
+                    .fetch_optional(&db)
+                    .await
+                    .unwrap();
+            assert_eq!(
+                running,
+                Some(false),
+                "a paused worker must not pull the job"
+            );
+
+            set_paused(false);
+            while completed.next().await != Some(id) {}
+        },
+        port,
+    )
+    .await;
+
+    assert!(completed_job(id, &db).await.success);
+    Ok(())
+}

backend/tests/worker_ping_ip.rs

@@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres};
 use uuid::Uuid;
 use windmill_common::{
     external_ip::UNKNOWN_IP,
-    worker::{insert_ping_query, update_worker_ping_main_loop_query},
+    worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query},
 };
 
 async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> anyhow::Result<()> {
+    register(db, worker, ip, None).await
+}
+
+async fn register(
+    db: &Pool<Postgres>,
+    worker: &str,
+    ip: Option<&str>,
+    draining: Option<bool>,
+) -> anyhow::Result<()> {
     insert_ping_query(
         "test-instance",
         worker,
@@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> any
         None,
         None,
         false,
+        draining,
         db,
     )
     .await?;
     Ok(())
 }
 
+/// A worker re-registers mid-life once its external IP resolves. That registration must not
+/// clear a draining flag the same process already published; only a worker starting up under
+/// a reclaimed name resets it.
+#[sqlx::test]
+async fn only_a_starting_worker_resets_draining(db: Pool<Postgres>) -> anyhow::Result<()> {
+    let draining = || async {
+        sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'")
+            .fetch_one(&db)
+            .await
+    };
+    register(&db, "wk-drain", None, Some(false)).await?;
+    set_worker_draining_query("wk-drain", &db).await?;
+    register(&db, "wk-drain", Some("1.2.3.4"), None).await?;
+    assert!(draining().await?);
+
+    register(&db, "wk-drain", None, Some(false)).await?;
+    assert!(!draining().await?);
+    Ok(())
+}
+
 /// The external IP resolves in the background, so the initial ping often has none yet. That must
 /// not blank the address a previous process wrote to the row this one reclaims — worker names are
 /// stable across restarts under EXIT_AFTER_N_JOBS.

backend/windmill-ai/src/ai_providers.rs

@@ -124,7 +124,7 @@ pub enum AIProvider {
     #[serde(rename = "aws_bedrock")]
     AWSBedrock,
     /// TypeSafe's Jev decision model. It answers typed questions rather than chatting, so only
-    /// an AI decision runs it (`run_systemone`), never the agent loop.
+    /// an AI decision runs it (`run_decision`), never the agent loop.
     TypeSafe,
     /// Cloudflare Workers AI, for its Jev-compatible decision models (Clef). Decisions only, as
     /// for TypeSafe; its base URL comes from the resource's account id.
@@ -152,6 +152,12 @@ impl AIProvider {
         matches!(self, AIProvider::TypeSafe | AIProvider::Cloudflare)
     }
 
+    /// A provider an AI decision runs on: the decision providers, and OpenAI, whose Decisions
+    /// API sits next to its chat routes on the same resource.
+    pub fn runs_decisions(&self) -> bool {
+        self.is_decision_provider() || matches!(self, AIProvider::OpenAI)
+    }
+
     /// Get the base URL for the AI provider
     pub async fn get_base_url(&self, resource_base_url: Option<String>, db: &DB) -> Result<String> {
         if let Some(base_url) = resource_base_url {

backend/windmill-ai/src/decision.rs

@@ -1,6 +1,7 @@
-//! AI decision steps: one call to a System One endpoint (TypeSafe's Jev, or Cloudflare's
-//! Jev-compatible Clef on Workers AI), which answers typed questions (`choice`, `score`, `noul`)
-//! about a state with calibrated probabilities instead of text.
+//! AI decision steps: one call to a decision endpoint, which answers typed questions (`choice`,
+//! `score`, `noul`) about a state with calibrated probabilities instead of text. The questions and
+//! answers are System One's (TypeSafe's Jev, Cloudflare's Jev-compatible Clef on Workers AI);
+//! OpenAI's Decisions API is asked and read through a translation to that shape.
 
 use std::time::Duration;
 
@@ -30,6 +31,8 @@ pub struct AIDecisionArgs {
 pub const TYPESAFE_DEFAULT_MODEL: &str = "jev-latest";
 /// Cloudflare's larger decision model, run when the step names none.
 pub const CLOUDFLARE_DEFAULT_MODEL: &str = "clef";
+/// The model OpenAI's Decisions API serves, run when the step names none.
+pub const OPENAI_DEFAULT_MODEL: &str = "gpt-6-luna";
 
 #[derive(Serialize)]
 struct SystemOneRequest<'a> {
@@ -108,7 +111,263 @@ pub fn decision_inputs<'a>(
     Ok((state, questions))
 }
 
-pub async fn run_systemone(
+/// Posts a decision request with the resource's bearer key and returns the response body.
+async fn post_decision<T: Serialize>(
+    credentials: &ProviderCredentials,
+    endpoint: &str,
+    vendor: &str,
+    payload: &T,
+    timeout: Duration,
+) -> Result<String> {
+    let auth_headers = retain_effective_credentials(
+        credentials,
+        vec![(
+            "Authorization",
+            format!(
+                "Bearer {}",
+                credentials.api_key.as_deref().unwrap_or_default()
+            ),
+        )],
+    );
+
+    // The endpoint derives from the resource's base URL, so the connect is pinned to the
+    // address the SSRF check validated.
+    let client = pinned_ai_client_for(&credentials.base_url).await?;
+    let mut request = client.post(endpoint).timeout(timeout);
+    for (name, value) in auth_headers {
+        request = request.header(name, value);
+    }
+    for (name, value) in common_outbound_headers(credentials) {
+        request = request.header(name, value);
+    }
+
+    let response = request
+        .json(payload)
+        .send()
+        .await
+        .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?;
+
+    let status = response.status();
+    if !status.is_success() {
+        let body = response
+            .text()
+            .await
+            .unwrap_or_else(|_| "<failed to read body>".to_string());
+        return Err(Error::ExecutionErr(format!(
+            "{vendor} error calling {endpoint}: {status} - {body}"
+        )));
+    }
+
+    response
+        .text()
+        .await
+        .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}")))
+}
+
+/// Answers a decision on the endpoint its provider serves.
+pub async fn run_decision(
+    credentials: &ProviderCredentials,
+    model: &str,
+    state: &Value,
+    questions: &Map<String, Value>,
+    timeout: Duration,
+) -> Result<DecisionResult> {
+    if credentials.provider == AIProvider::OpenAI {
+        run_openai_decisions(credentials, model, state, questions, timeout).await
+    } else {
+        run_systemone(credentials, model, state, questions, timeout).await
+    }
+}
+
+#[derive(Serialize)]
+struct OpenAIDecisionsRequest<'a> {
+    model: &'a str,
+    input: Value,
+    questions: Vec<Value>,
+}
+
+#[derive(Deserialize)]
+struct OpenAIDecisionsResponse {
+    #[serde(default)]
+    model: Option<String>,
+    answers: Vec<Map<String, Value>>,
+    #[serde(default)]
+    usage: Option<SystemOneUsage>,
+}
+
+/// The state as a Decisions `input`. OpenAI reads a text or a list of user messages, so a list of
+/// user messages (the only way to pass an image) goes as it is and any other object or array,
+/// a chat history included, as its JSON text.
+fn openai_input(state: &Value) -> Value {
+    match state {
+        Value::String(_) => state.clone(),
+        Value::Array(items) if !items.is_empty() && items.iter().all(is_openai_user_message) => {
+            state.clone()
+        }
+        _ => Value::String(state.to_string()),
+    }
+}
+
+/// Exactly a message the Decisions API reads: `role: "user"` with a text, or with parts that are
+/// each an `input_text` or an `input_image`.
+fn is_openai_user_message(message: &Value) -> bool {
+    let is_part = |part: &Value| match part.get("type").and_then(Value::as_str) {
+        Some("input_text") => part.get("text").is_some_and(Value::is_string),
+        Some("input_image") => part.get("image_url").is_some_and(Value::is_string),
+        _ => false,
+    };
+    message.get("role").and_then(Value::as_str) == Some("user")
+        && match message.get("content") {
+            Some(Value::String(_)) => true,
+            Some(Value::Array(parts)) => !parts.is_empty() && parts.iter().all(is_part),
+            _ => false,
+        }
+}
+
+/// The questions as OpenAI's Decisions API takes them: a list naming each question, with a
+/// `noul` asked as its `predicate` and each score level as a label.
+fn openai_questions(questions: &Map<String, Value>) -> Result<Vec<Value>> {
+    let text = |v: &Value| match v {
+        Value::String(s) => s.clone(),
+        v => v.to_string(),
+    };
+    // A structured description goes as its JSON text, as structured instructions do.
+    let stated = |description: Option<&Value>| match description {
+        None | Some(Value::Null) => None,
+        Some(Value::String(d)) if d.trim().is_empty() => None,
+        Some(d) => Some(text(d)),
+    };
+    questions
+        .iter()
+        .map(|(name, question)| {
+            let criteria = question.get("criteria");
+            let mut instructions = question.get("instructions").map(text).unwrap_or_default();
+            let mut asked = Map::new();
+            let kind = match question.get("type").and_then(Value::as_str) {
+                Some("choice") => {
+                    let choices = criteria
+                        .and_then(Value::as_object)
+                        .filter(|c| !c.is_empty())
+                        .ok_or_else(|| {
+                            Error::BadRequest(format!(
+                                "The choice question '{name}' needs its options as 'criteria'"
+                            ))
+                        })?
+                        .iter()
+                        .map(|(value, d)| match stated(Some(d)) {
+                            Some(d) => serde_json::json!({"value": value, "description": d}),
+                            None => serde_json::json!({"value": value}),
+                        })
+                        .collect();
+                    asked.insert("choices".to_string(), Value::Array(choices));
+                    "choice"
+                }
+                Some("score") => {
+                    let levels = criteria
+                        .and_then(Value::as_array)
+                        .filter(|c| !c.is_empty())
+                        .ok_or_else(|| {
+                            Error::BadRequest(format!(
+                                "The score question '{name}' needs its levels as 'criteria'"
+                            ))
+                        })?
+                        .iter()
+                        .map(|level| serde_json::json!({"label": text(level)}))
+                        .collect();
+                    asked.insert("levels".to_string(), Value::Array(levels));
+                    "score"
+                }
+                Some("noul") => {
+                    for (key, meaning) in [("true", "Yes"), ("false", "No")] {
+                        if let Some(d) = stated(criteria.and_then(|c| c.get(key))) {
+                            instructions.push_str(&format!("\n{meaning}: {d}"));
+                        }
+                    }
+                    "predicate"
+                }
+                other => {
+                    return Err(Error::BadRequest(format!(
+                    "The question '{name}' has type {}, which is not 'choice', 'score' or 'noul'",
+                    other.map_or("none".to_string(), |t| format!("'{t}'"))
+                )))
+                }
+            };
+            asked.insert("type".to_string(), kind.into());
+            asked.insert("name".to_string(), name.as_str().into());
+            asked.insert("instructions".to_string(), instructions.into());
+            Ok(Value::Object(asked))
+        })
+        .collect()
+}
+
+/// OpenAI's answers keyed by question name, as every decision's are. A predicate's probability is
+/// also given as `noul`, so a flow reads a yes/no answer the same way whichever provider gave it.
+/// A refused question fails the decision: left in the output it has no `choice` or `noul`, so a
+/// branch on it would quietly take its "no" or default arm.
+fn openai_answers(answers: Vec<Map<String, Value>>) -> Result<Map<String, Value>> {
+    let mut by_name = Map::new();
+    for mut answer in answers {
+        let Some(Value::String(name)) = answer.remove("name") else {
+            continue;
+        };
+        match answer.get("type").and_then(Value::as_str) {
+            Some("refusal") => {
+                return Err(Error::ExecutionErr(format!(
+                    "OpenAI refused to answer the question '{name}': {}",
+                    Value::Object(answer)
+                )))
+            }
+            Some("predicate") => {
+                if let Some(probability) = answer.get("probability").cloned() {
+                    answer.insert("noul".to_string(), probability);
+                }
+            }
+            _ => {}
+        }
+        by_name.insert(name, Value::Object(answer));
+    }
+    Ok(by_name)
+}
+
+async fn run_openai_decisions(
+    credentials: &ProviderCredentials,
+    model: &str,
+    state: &Value,
+    questions: &Map<String, Value>,
+    timeout: Duration,
+) -> Result<DecisionResult> {
+    let model = match model.trim() {
+        "" => OPENAI_DEFAULT_MODEL,
+        model => model,
+    };
+    let endpoint = format!("{}/decisions", credentials.base_url.trim_end_matches('/'));
+    let body = post_decision(
+        credentials,
+        &endpoint,
+        "OpenAI",
+        &OpenAIDecisionsRequest {
+            model,
+            input: openai_input(state),
+            questions: openai_questions(questions)?,
+        },
+        timeout,
+    )
+    .await?;
+    let parsed = serde_json::from_str::<OpenAIDecisionsResponse>(&body)
+        .map_err(|e| Error::ExecutionErr(format!("Unexpected OpenAI response ({e}): {body}")))?;
+
+    Ok(DecisionResult {
+        output: serde_json::value::to_raw_value(&openai_answers(parsed.answers)?)
+            .map_err(|e| Error::internal_err(format!("serializing decision answers: {e}")))?,
+        model: parsed.model,
+        usage: parsed
+            .usage
+            .map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens))
+            .filter(|u| !u.is_empty()),
+    })
+}
+
+async fn run_systemone(
     credentials: &ProviderCredentials,
     model: &str,
     state: &Value,
@@ -138,49 +397,14 @@ pub async fn run_systemone(
         format!("{base}/systemone")
     };
     let vendor = if cloudflare { "Cloudflare" } else { "TypeSafe" };
-    let auth_headers = retain_effective_credentials(
+    let body = post_decision(
         credentials,
-        vec![(
-            "Authorization",
-            format!(
-                "Bearer {}",
-                credentials.api_key.as_deref().unwrap_or_default()
-            ),
-        )],
-    );
-
-    // The endpoint derives from the resource's base URL, so the connect is pinned to the
-    // address the SSRF check validated.
-    let client = pinned_ai_client_for(&credentials.base_url).await?;
-    let mut request = client.post(&endpoint).timeout(timeout);
-    for (name, value) in auth_headers {
-        request = request.header(name, value);
-    }
-    for (name, value) in common_outbound_headers(credentials) {
-        request = request.header(name, value);
-    }
-
-    let response = request
-        .json(&SystemOneRequest { model, state, questions })
-        .send()
-        .await
-        .map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?;
-
-    let status = response.status();
-    if !status.is_success() {
-        let body = response
-            .text()
-            .await
-            .unwrap_or_else(|_| "<failed to read body>".to_string());
-        return Err(Error::ExecutionErr(format!(
-            "{vendor} error calling {endpoint}: {status} - {body}"
-        )));
-    }
-
-    let body = response
-        .text()
-        .await
-        .map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}")))?;
+        &endpoint,
+        vendor,
+        &SystemOneRequest { model, state, questions },
+        timeout,
+    )
+    .await?;
     // Tried in turn rather than as an untagged enum, which cannot hold the raw `answers`.
     let parsed = match serde_json::from_str::<SystemOneResponse>(&body) {
         Ok(parsed) => parsed,
@@ -223,6 +447,80 @@ mod tests {
         }
     }
 
+    /// A `noul` goes out as a predicate with its yes/no meanings in the instructions, and comes
+    /// back readable as `noul`; choices and levels go out as lists, answers come back by name.
+    #[test]
+    fn openai_questions_and_answers_translate() {
+        let questions = json!({
+            "intent": {"type": "choice", "instructions": "Why?", "criteria": {"refund": {"when": "paid"}, "bug": ""}},
+            "urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["Can wait", "Now"]},
+            "angry": {"type": "noul", "instructions": "Angry?", "criteria": {"true": "Insults"}}
+        });
+        let asked = Value::Array(openai_questions(questions.as_object().unwrap()).unwrap());
+        assert_eq!(
+            asked,
+            json!([
+                {"choices": [
+                    {"value": "refund", "description": "{\"when\":\"paid\"}"},
+                    {"value": "bug"}
+                ], "type": "choice", "name": "intent", "instructions": "Why?"},
+                {"levels": [
+                    {"label": "Can wait"},
+                    {"label": "Now"}
+                ], "type": "score", "name": "urgency", "instructions": "How urgent?"},
+                {"type": "predicate", "name": "angry", "instructions": "Angry?\nYes: Insults"}
+            ])
+        );
+        for bad in [
+            json!({"q": {"type": "choice", "instructions": "?"}}),
+            json!({"q": {"type": "score", "instructions": "?", "criteria": []}}),
+            json!({"q": {"type": "predicate", "instructions": "?"}}),
+        ] {
+            assert!(openai_questions(bad.as_object().unwrap()).is_err(), "{bad}");
+        }
+
+        let answers = serde_json::from_value(json!([
+            {"type": "predicate", "name": "angry", "probability": 0.92},
+            {"type": "choice", "name": "intent", "choice": "refund", "confidence": 0.9}
+        ]))
+        .unwrap();
+        assert_eq!(
+            Value::Object(openai_answers(answers).unwrap()),
+            json!({
+                "angry": {"type": "predicate", "probability": 0.92, "noul": 0.92},
+                "intent": {"type": "choice", "choice": "refund", "confidence": 0.9}
+            })
+        );
+
+        let messages = json!([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]);
+        assert_eq!(openai_input(&messages), messages);
+        assert_eq!(openai_input(&json!({"m": "hi"})), json!("{\"m\":\"hi\"}"));
+        assert_eq!(
+            openai_input(&json!([{"role": "admin"}])),
+            json!("[{\"role\":\"admin\"}]")
+        );
+        for records in [
+            json!([{"role": "user", "content": {"ticket": "charged twice"}}]),
+            json!([{"role": "user", "content": null}]),
+            json!([{"role": "user", "content": ["a"]}]),
+            json!([{"role": "user", "content": [{"type": "input_text", "text": {"t": 1}}]}]),
+            json!([{"role": "user", "content": [{"type": "input_image"}]}]),
+        ] {
+            assert_eq!(openai_input(&records), json!(records.to_string()));
+        }
+        assert_eq!(
+            openai_input(&json!([{"role": "user", "content": "Hi"}])),
+            json!([{"role": "user", "content": "Hi"}])
+        );
+        // A chat history is evaluated as text: OpenAI takes user messages only.
+        let history =
+            json!([{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Yo"}]);
+        assert_eq!(openai_input(&history), json!(history.to_string()));
+        let refused =
+            serde_json::from_value(json!([{"type": "refusal", "name": "angry"}])).unwrap();
+        assert!(openai_answers(refused).is_err());
+    }
+
     /// Refused are the values a flow produces when it forgot the input (no key, `null`, a blank
     /// string from an empty expression) and the scalars TypeSafe rejects. A text, an object or an
     /// array is a state, whatever it holds.

backend/windmill-ai/src/providers/mod.rs

@@ -62,9 +62,13 @@ const REASONING_RULES: &[ReasoningRule] = &[
         can_disable: false,
         completions_tools_need_off: false,
     },
-    // Live-verified: astra takes low..max only, where sol and luna also take `none`.
+    // Live-verified: gpt-6-astra and gpt-6.1-sol take low..max only, where gpt-6-sol and
+    // gpt-6-luna also take `none`.
     ReasoningRule {
-        matches: |m| base_id(m).starts_with("gpt-6-astra"),
+        matches: |m| {
+            let base = base_id(m);
+            base.starts_with("gpt-6-astra") || base.starts_with("gpt-6.1-sol")
+        },
         can_disable: false,
         completions_tools_need_off: true,
     },

backend/windmill-api-auth/src/scopes.rs

@@ -996,6 +996,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui
         ["job_metrics", "set_progress", ..] => (http_method == "POST", 2),
         ["job_metrics", "get_progress", ..] => (http_method == "GET", 2),
         ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2),
+        ["jobs", "worker_is_draining", ..] => (http_method == "GET", 2),
         ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3),
         ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3),
         _ => return false,

backend/windmill-api-configs/src/lib.rs

@@ -224,12 +224,13 @@ async fn update_config(
 
     #[cfg(not(feature = "enterprise"))]
     let config = if name.starts_with("worker__") {
-        // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode
+        // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused
         serde_json::json!({
             "worker_tags": config.get("worker_tags"),
             "cache_clear": config.get("cache_clear"),
             "init_bash": config.get("init_bash"),
-            "native_mode": config.get("native_mode")
+            "native_mode": config.get("native_mode"),
+            "paused": config.get("paused")
         })
     } else {
         config
@@ -272,6 +273,18 @@ async fn update_config(
     }
 
     let mut tx = db.begin().await?;
+    let was_paused = if name.starts_with("worker__") {
+        sqlx::query_scalar!(
+            "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE",
+            &name
+        )
+        .fetch_optional(&mut *tx)
+        .await?
+        .flatten()
+        .unwrap_or(false)
+    } else {
+        false
+    };
     sqlx::query!(
         "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config",
         &name,
@@ -291,6 +304,17 @@ async fn update_config(
     )
     .await?;
     tx.commit().await?;
+
+    if name.starts_with("worker__") {
+        let paused = config.get("paused").and_then(|v| v.as_bool()) == Some(true);
+        if paused != was_paused {
+            windmill_common::feature_usage::log_feature_usage(
+                "worker_group",
+                "pause_toggle",
+                if paused { "pause" } else { "resume" },
+            );
+        }
+    }
     Ok(format!("Updated config {name}"))
 }
 

backend/windmill-api-integration-tests/tests/datatable_roles.rs

@@ -417,6 +417,89 @@ async fn concurrent_role_creations_both_survive(db: Pool<Postgres>) -> anyhow::R
     outcome
 }
 
+#[cfg(all(feature = "private", feature = "enterprise"))]
+#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
+async fn an_existing_role_is_taken_over_only_when_asked_and_never_with_members(
+    db: Pool<Postgres>,
+) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string();
+    let plain = format!("wmtest_plain_{suffix}");
+    let group = format!("wmtest_group_{suffix}");
+    let member = format!("wmtest_member_{suffix}");
+    for sql in [
+        // An expired password and no connection slot: kept as is, the login Windmill sets is dead.
+        format!(
+            "CREATE ROLE \"{plain}\" LOGIN PASSWORD 'theirs' VALID UNTIL '2000-01-01' \
+             CONNECTION LIMIT 0"
+        ),
+        format!("CREATE ROLE \"{group}\" NOLOGIN"),
+        format!("CREATE ROLE \"{member}\" LOGIN"),
+        format!("GRANT \"{group}\" TO \"{member}\""),
+    ] {
+        sqlx::query(&sql).execute(&db).await?;
+    }
+
+    let create = |name: String, take_over: bool| async move {
+        let resp = authed(
+            client().post(format!(
+                "http://localhost:{port}/api/settings/datatable_roles"
+            )),
+            "SECRET_TOKEN",
+        )
+        .json(&json!({ "name": name, "take_over": take_over }))
+        .send()
+        .await?;
+        Ok::<_, anyhow::Error>((resp.status().as_u16(), resp.text().await?))
+    };
+    let outcome = async {
+        let (status, body) = create(plain.clone(), false).await?;
+        assert_eq!(
+            status, 409,
+            "an existing role is not taken without asking: {body}"
+        );
+
+        // The group's member would inherit every grant the data table role gets.
+        let (status, body) = create(group.clone(), true).await?;
+        assert_eq!(status, 400, "{body}");
+        assert!(
+            body.contains(&member),
+            "the refusal names the member: {body}"
+        );
+
+        let (status, body) = create(plain.clone(), true).await?;
+        assert_eq!(status, 200, "{body}");
+        let catalog = windmill_common::datatable_roles::read_role_catalog(
+            &db,
+            windmill_common::datatable_roles::DatatableRoleCluster::Instance,
+        )
+        .await?;
+        let taken = catalog
+            .values()
+            .find(|r| r.name == plain)
+            .ok_or_else(|| anyhow::anyhow!("{plain} is not in the catalog: {catalog:?}"))?;
+        let options = (*db.connect_options())
+            .clone()
+            .username(&plain)
+            .password(taken.pwd.as_deref().unwrap_or_default());
+        let mut login = <sqlx::PgConnection as sqlx::Connection>::connect_with(&options).await?;
+        sqlx::query("SELECT 1").execute(&mut login).await?;
+        Ok::<_, anyhow::Error>(())
+    }
+    .await;
+
+    // Roles are cluster-wide and outlive this test's database, whatever happened above.
+    for name in [&member, &group, &plain] {
+        let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\""))
+            .execute(&db)
+            .await;
+    }
+    outcome
+}
+
 #[cfg(all(feature = "private", feature = "enterprise"))]
 #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))]
 async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled(

backend/windmill-api-integration-tests/tests/jobs_authed.rs

@@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue(
 
     Ok(())
 }
+
+#[sqlx::test(migrations = "../migrations", fixtures("base"))]
+async fn test_worker_is_draining_follows_the_job_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await?;
+    let port = server.addr.port();
+
+    let job = Uuid::new_v4();
+    sqlx::query(
+        "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args)
+         VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)",
+    )
+    .bind(job)
+    .execute(&db)
+    .await?;
+    sqlx::query(
+        "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker)
+         VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')",
+    )
+    .bind(job)
+    .execute(&db)
+    .await?;
+    sqlx::query(
+        "INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version)
+         VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')",
+    )
+    .execute(&db)
+    .await?;
+
+    let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}");
+    let draining = || async {
+        let resp = authed(client().get(&url)).send().await?;
+        anyhow::Ok(resp.json::<bool>().await?)
+    };
+    assert!(!draining().await?);
+
+    windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?;
+    assert!(draining().await?);
+
+    // A parked job keeps its last worker on the queue row without running on it.
+    sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1")
+        .bind(job)
+        .execute(&db)
+        .await?;
+    assert!(!draining().await?);
+
+    Ok(())
+}

backend/windmill-api-jobs/Cargo.toml

@@ -15,6 +15,7 @@ enterprise = ["windmill-common/enterprise"]
 [dependencies]
 windmill-api-auth.workspace = true
 windmill-api-sse.workspace = true
+windmill-audit.workspace = true
 windmill-common = { workspace = true, default-features = false }
 windmill-queue.workspace = true
 axum.workspace = true

backend/windmill-api-jobs/src/execution.rs

@@ -23,12 +23,17 @@ use std::sync::Arc;
 use tokio::sync::RwLock;
 use uuid::Uuid;
 use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed};
+use windmill_audit::{
+    audit_oss::{audit_log_many, AuditAuthorable},
+    ActionKind,
+};
 use windmill_common::{
     db::{UserDB, UserDbWithAuthed},
     error::{self, Error},
     flow_conversations::{
         add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType,
     },
+    flows::Retry,
     get_latest_flow_version_info_for_path,
     jobs::{
         check_tag_available_for_workspace_internal, format_result, script_path_to_payload,
@@ -122,6 +127,71 @@ pub async fn drop_unclaimable_run_lineage(
     Ok(())
 }
 
+/// Applies the run's `retry` policy by handing `push` a one-step-flow request, which it
+/// materializes as a native retryable `Script`.
+pub fn with_run_retry(
+    run_query: &RunJobQuery,
+    payload: JobPayload,
+    args: &PushArgs<'_>,
+    tag: &Option<String>,
+) -> error::Result<JobPayload> {
+    let Some(retry) = run_query.retry.as_deref() else {
+        return Ok(payload);
+    };
+    // Retry attempts are always pushed visible to the owner, which would leak a private run.
+    if run_query.invisible_to_owner.unwrap_or(false) {
+        return Err(Error::BadRequest(
+            "retry cannot be combined with invisible_to_owner".to_string(),
+        ));
+    }
+    let retry: Retry = serde_json::from_str(retry)
+        .map_err(|e| Error::BadRequest(format!("invalid retry policy: {e}")))?;
+    let JobPayload::ScriptHash {
+        hash,
+        path,
+        cache_ttl,
+        cache_ignore_s3_path,
+        language,
+        priority,
+        apply_preprocessor,
+        debouncing_settings,
+        concurrency_settings,
+        ..
+    } = payload
+    else {
+        return Err(Error::BadRequest(
+            "retry is only supported for workspace scripts".to_string(),
+        ));
+    };
+    // A failed attempt keeps its raw args and the retry re-push skips the preprocessor, so
+    // every retry would call `main` with un-preprocessed input.
+    if apply_preprocessor {
+        return Err(Error::BadRequest(
+            "retry is not supported for a script with a preprocessor unless skip_preprocessor is set"
+                .to_string(),
+        ));
+    }
+    Ok(JobPayload::SingleStepFlow {
+        path,
+        hash: Some(hash),
+        flow_version: None,
+        language: Some(language),
+        args: HashMap::from(args),
+        retry: Some(retry),
+        error_handler_path: None,
+        error_handler_args: None,
+        skip_handler: None,
+        cache_ttl,
+        cache_ignore_s3_path,
+        priority,
+        tag_override: tag.clone(),
+        trigger_path: None,
+        apply_preprocessor,
+        concurrency_settings,
+        debouncing_settings,
+    })
+}
+
 /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the
 /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for
 /// a workspace admin anything outside the workspace. A restricted job token never gets the admin
@@ -196,11 +266,15 @@ pub async fn check_license_key_valid() -> error::Result<()> {
 pub async fn cancel_jobs(
     jobs: Vec<Uuid>,
     db: &DB,
-    username: &str,
+    author: &(impl AuditAuthorable + Sync),
     w_id: &str,
     force_cancel: bool,
 ) -> error::JsonResult<Vec<Uuid>> {
+    let username = author.username();
     let mut uuids = vec![];
+    // A job that completed before its turn is reported back like the others but was not
+    // cancelled, so it gets no audit entry.
+    let mut audited = vec![];
     tracing::info!("Cancelling jobs: {:?}", jobs);
     let mut tx = db.begin().await?;
     let trivial_jobs =  sqlx::query!("INSERT INTO v2_job_completed AS cj
@@ -246,7 +320,7 @@ pub async fn cancel_jobs(
         }
         match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move {
             let tx = db.begin().await?;
-            let (tx, _) = cancel_job(
+            let (tx, cancelled) = cancel_job(
                 username,
                 None,
                 job_id.clone(),
@@ -258,13 +332,14 @@ pub async fn cancel_jobs(
             )
             .await?;
             tx.commit().await?;
-            Ok::<_, anyhow::Error>(())
+            Ok::<_, anyhow::Error>(cancelled)
         })
         .await
         {
             Ok(result) => match result {
-                Ok(_) => {
+                Ok(cancelled) => {
                     uuids.push(job_id);
+                    audited.extend(cancelled);
                 }
                 Err(e) => {
                     tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e);
@@ -279,8 +354,27 @@ pub async fn cancel_jobs(
         }
     }
 
+    audited.extend(trivial_jobs.iter().copied());
     uuids.extend(trivial_jobs);
 
+    // The cancels are committed by now: failing here would report them as not done.
+    if let Err(e) = audit_log_many(
+        db,
+        author,
+        if force_cancel {
+            "jobs.force_cancel"
+        } else {
+            "jobs.cancel"
+        },
+        ActionKind::Delete,
+        w_id,
+        &audited.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
+    )
+    .await
+    {
+        tracing::error!("Failed to write audit entries for cancelled jobs in {w_id}: {e:#}");
+    }
+
     Ok(Json(uuids))
 }
 
@@ -1123,6 +1217,7 @@ pub async fn push_script_job_by_path_into_queue<'c>(
     let tag = run_query.tag.clone().or(tag);
     let push_args = PushArgs { args: &args.args, extra: args.extra };
     check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?;
+    let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?;
 
     let return_tx = tx_o.is_some();
 

backend/windmill-api-jobs/src/types.rs

@@ -52,6 +52,8 @@ pub struct RunJobQuery {
     pub trigger_external_id: Option<String>,
     pub service_name: Option<String>,
     pub suspended_mode: Option<bool>,
+    /// A JSON-encoded `Retry` policy for a script run; see `with_run_retry`.
+    pub retry: Option<String>,
 }
 
 impl RunJobQuery {

backend/windmill-api-schedule/src/lib.rs

@@ -1487,11 +1487,12 @@ async fn set_default_error_handler(
         }
     };
 
-    if let Some(value_content) = value {
-        windmill_api_settings::set_global_setting_internal(&db, key, value_content).await?;
+    if let Some(value_content) = value.clone() {
+        windmill_api_settings::set_global_setting_internal(&db, key.clone(), value_content).await?;
     } else {
         windmill_api_settings::delete_global_setting(&db, key.as_str()).await?;
     }
+    windmill_api_settings::audit_setting_write(&db, &authed, &key, value.as_ref()).await?;
 
     if payload.override_existing {
         // The rewrite and its history rows go in one transaction: on separate

backend/windmill-api-settings/src/lib.rs

@@ -29,6 +29,7 @@ mod log_cleanup;
 mod storage_usage;
 
 use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed};
+use windmill_audit::{audit_oss::audit_log, ActionKind};
 use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT;
 use windmill_common::DB;
 
@@ -829,7 +830,71 @@ pub async fn set_global_setting(
     Json(value): Json<Value>,
 ) -> error::Result<()> {
     require_super_admin(&db, &authed).await?;
-    set_global_setting_internal(&db, key, value.value.unwrap_or(serde_json::Value::Null)).await
+    let value = value.value.unwrap_or(serde_json::Value::Null);
+    set_global_setting_internal(&db, key.clone(), value.clone()).await?;
+    audit_setting_write(&db, &authed, &key, Some(&value)).await
+}
+
+/// A setting is recorded by name, with its value only when that is a boolean or a number.
+/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no
+/// list of secret-bearing settings stays complete as settings are added.
+///
+/// AUTHORIZATION: records a write, checks nothing. The caller must have established that
+/// `authed` may change instance settings before performing the write this records.
+pub async fn audit_setting_write(
+    db: &DB,
+    authed: &ApiAuthed,
+    key: &str,
+    value: Option<&serde_json::Value>,
+) -> error::Result<()> {
+    let deleted = match value {
+        None | Some(serde_json::Value::Null) => true,
+        Some(serde_json::Value::String(s)) => s.trim().is_empty(),
+        Some(_) => false,
+    };
+    let recorded = match value {
+        Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => {
+            Some(v.to_string())
+        }
+        _ => None,
+    };
+    audit_log(
+        db,
+        authed,
+        if deleted {
+            "global_settings.delete"
+        } else {
+            "global_settings.update"
+        },
+        if deleted {
+            ActionKind::Delete
+        } else {
+            ActionKind::Update
+        },
+        "global",
+        Some(key),
+        recorded.as_deref().map(|v| [("value", v)].into()),
+    )
+    .await
+}
+
+async fn audit_instance_action(
+    db: &DB,
+    authed: &ApiAuthed,
+    operation: &str,
+    action_kind: ActionKind,
+    parameters: Option<std::collections::HashMap<&str, &str>>,
+) -> error::Result<()> {
+    audit_log(
+        db,
+        authed,
+        operation,
+        action_kind,
+        "global",
+        Some(&authed.email),
+        parameters,
+    )
+    .await
 }
 
 pub async fn set_global_setting_internal(
@@ -1305,15 +1370,61 @@ async fn set_instance_config(
                 run_setting_pre_write_hook(&db, key, value).await?;
             }
         }
+        // The external-instance write removes its own key from the diff.
+        let external = settings_diff
+            .upserts
+            .get(EXTERNAL_INSTANCE_PG_SETTING)
+            .cloned()
+            .map(Some)
+            .or_else(|| {
+                settings_diff
+                    .deletes
+                    .iter()
+                    .any(|k| k == EXTERNAL_INSTANCE_PG_SETTING)
+                    .then_some(None)
+            });
         windmill_common::external_instance_pg::write_external_instance_pg_from_diff(
             &db,
             &mut settings_diff,
         )
         .await?;
+        if let Some(value) = external {
+            audit_setting_write(&db, &authed, EXTERNAL_INSTANCE_PG_SETTING, value.as_ref()).await?;
+        }
 
-        instance_config::apply_settings_diff(&db, &settings_diff)
-            .await
-            .map_err(|e| error::Error::internal_err(e.to_string()))?;
+        // Applied and audited one setting at a time: the writes are not transactional, so a
+        // batch that fails midway must not leave the settings it did write unrecorded.
+        let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } =
+            settings_diff;
+        let steps = upserts
+            .into_iter()
+            .map(|(k, v)| (k, Some(v)))
+            .chain(deletes.into_iter().map(|k| (k, None)))
+            .map(Some)
+            .chain(std::iter::once(None));
+        for step in steps {
+            let mut diff = instance_config::SettingsDiff::default();
+            match &step {
+                Some((key, value)) => {
+                    if let Some(previous) = previous_values.remove(key) {
+                        diff.previous_values.insert(key.clone(), previous);
+                    }
+                    match value {
+                        Some(value) => {
+                            diff.upserts.insert(key.clone(), value.clone());
+                        }
+                        None => diff.deletes.push(key.clone()),
+                    }
+                }
+                None => diff.unchanged_count = unchanged_count,
+            }
+            instance_config::apply_settings_diff(&db, &diff)
+                .await
+                .map_err(|e| error::Error::internal_err(e.to_string()))?;
+            if let Some((key, value)) = &step {
+                audit_setting_write(&db, &authed, key, value.as_ref()).await?;
+            }
+        }
 
         if ai_config_changed {
             bump_instance_ai_config_revision();
@@ -1343,9 +1454,37 @@ async fn set_instance_config(
             .collect();
         let configs_diff =
             instance_config::diff_worker_configs(&current_wc, &desired_wc, ApplyMode::Merge);
-        instance_config::apply_configs_diff(&db, &configs_diff)
-            .await
-            .map_err(|e| error::Error::internal_err(e.to_string()))?;
+        let steps = configs_diff
+            .upserts
+            .into_iter()
+            .map(|(g, c)| (g, Some(c)))
+            .chain(configs_diff.deletes.into_iter().map(|g| (g, None)));
+        for (group, config) in steps {
+            let mut diff = instance_config::ConfigsDiff::default();
+            let (operation, action_kind) = match config {
+                Some(config) => {
+                    diff.upserts.insert(group.clone(), config);
+                    ("worker_config.update", ActionKind::Update)
+                }
+                None => {
+                    diff.deletes.push(group.clone());
+                    ("worker_config.delete", ActionKind::Delete)
+                }
+            };
+            instance_config::apply_configs_diff(&db, &diff)
+                .await
+                .map_err(|e| error::Error::internal_err(e.to_string()))?;
+            audit_log(
+                &db,
+                &authed,
+                operation,
+                action_kind,
+                "global",
+                Some(&format!("worker__{group}")),
+                None,
+            )
+            .await?;
+        }
     }
 
     Ok(())
@@ -1583,6 +1722,14 @@ pub async fn renew_license_key(
             }
         )));
     } else {
+        audit_instance_action(
+            &db,
+            &authed,
+            "settings.renew_license_key",
+            ActionKind::Update,
+            None,
+        )
+        .await?;
         return Ok("Renewed license key".to_string());
     }
 }
@@ -1812,6 +1959,14 @@ async fn refresh_custom_instance_user_pwd(
     require_super_admin(&db, &authed).await?;
     windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?;
     windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?;
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.refresh_custom_instance_user_pwd",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(()))
 }
 
@@ -1970,7 +2125,8 @@ async fn setup_custom_instance_pg_database(
     windmill_common::datatable_roles::lock_instance_databases_governance(&mut tx, [dbname.trim()])
         .await?;
     let mut logs = CustomInstanceDbLogs::default();
-    let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
+    let result =
+        setup_custom_instance_pg_database_inner(authed.clone(), &db, &dbname, &mut logs).await;
     let success = result.is_ok();
     let error = result.err().map(|e| e.to_string());
     let status = CustomInstanceDb {
@@ -1998,6 +2154,20 @@ async fn setup_custom_instance_pg_database(
     .fetch_one(&mut *tx)
     .await?;
     tx.commit().await?;
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.setup_custom_instance_pg_database",
+        ActionKind::Create,
+        Some(
+            [
+                ("dbname", dbname.as_str()),
+                ("success", if success { "true" } else { "false" }),
+            ]
+            .into(),
+        ),
+    )
+    .await?;
     let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?;
 
     Ok(Json(status))
@@ -2176,6 +2346,14 @@ async fn drop_custom_instance_pg_database(
     require_super_admin(&db, &authed).await?;
 
     windmill_common::drop_custom_instance_database(&db, &dbname).await?;
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.drop_custom_instance_pg_database",
+        ActionKind::Delete,
+        Some([("dbname", dbname.as_str())].into()),
+    )
+    .await?;
 
     Ok(format!("Database '{}' dropped successfully", dbname))
 }
@@ -2220,6 +2398,14 @@ pub async fn migrate_secrets_to_vault(
 
     let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?;
 
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_to_vault",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 
@@ -2241,6 +2427,14 @@ pub async fn migrate_secrets_to_database(
     let report =
         windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?;
 
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_to_database",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 
@@ -2274,6 +2468,14 @@ pub async fn migrate_secrets_to_azure_kv(
     let report =
         windmill_common::secret_backend::migrate_secrets_to_azure_kv(&db, &settings).await?;
 
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_to_azure_kv",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 
@@ -2291,6 +2493,14 @@ pub async fn migrate_secrets_from_azure_kv(
     let report =
         windmill_common::secret_backend::migrate_secrets_from_azure_kv(&db, &settings).await?;
 
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_from_azure_kv",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 
@@ -2315,6 +2525,14 @@ pub async fn migrate_secrets_to_aws_sm(
 ) -> JsonResult<SecretMigrationReport> {
     require_super_admin(&db, &authed).await?;
     let report = windmill_common::secret_backend::migrate_secrets_to_aws_sm(&db, &settings).await?;
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_to_aws_sm",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 
@@ -2328,6 +2546,14 @@ pub async fn migrate_secrets_from_aws_sm(
     require_super_admin(&db, &authed).await?;
     let report =
         windmill_common::secret_backend::migrate_secrets_from_aws_sm(&db, &settings).await?;
+    audit_instance_action(
+        &db,
+        &authed,
+        "settings.migrate_secrets_from_aws_sm",
+        ActionKind::Update,
+        None,
+    )
+    .await?;
     Ok(Json(report))
 }
 

backend/windmill-api-workers/src/lib.rs

@@ -86,6 +86,7 @@ struct WorkerPing {
     job_isolation: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     native_mode: Option<bool>,
+    draining: bool,
 }
 
 // #[derive(Serialize, Deserialize)]
@@ -118,7 +119,7 @@ async fn list_worker_pings(
         WorkerPing,
         "SELECT worker, worker_instance,  EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
         CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,
-        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode
+        custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining
         FROM worker_ping
         WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
         ORDER BY ping_at desc LIMIT $2 OFFSET $3",

backend/windmill-api-workspaces/src/datatable_acl.rs

@@ -683,6 +683,9 @@ pub(crate) struct CatalogFacts {
     pub(crate) former_owner: Option<FormerOwnerDefaults>,
     /// For a revoke: each grant it takes back.
     pub(crate) revoked_grants: Vec<RevokedGrant>,
+    /// For a grant on what a schema holds: the role cannot reach the schema yet, so the grant
+    /// would give it nothing it can use.
+    pub(crate) lacks_schema_usage: bool,
 }
 
 /// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the
@@ -1526,7 +1529,24 @@ async fn build_plan(
             )));
         }
     }
-    let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants };
+    let lacks_schema_usage = match (&change, target) {
+        (AclChange::Grant { scope, .. }, AclTarget::Schema { schema })
+            if !matches!(scope, GrantScope::Target) =>
+        {
+            !has_schema_usage(client, schema, &pg_role).await?
+        }
+        (AclChange::Grant { .. }, AclTarget::Table { schema, .. }) => {
+            !has_schema_usage(client, schema, &pg_role).await?
+        }
+        _ => false,
+    };
+    let facts = CatalogFacts {
+        other_pg_roles,
+        existing_objects,
+        former_owner,
+        revoked_grants,
+        lacks_schema_usage,
+    };
     let mut plan =
         crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?;
     if matches!(change, AclChange::SetOwner { .. }) {
@@ -1540,6 +1560,27 @@ async fn build_plan(
     Ok(plan)
 }
 
+/// Whether `pg_role` may look into `schema`, directly or through a role it inherits from.
+async fn has_schema_usage(
+    client: &tokio_postgres::Client,
+    schema: &str,
+    pg_role: &str,
+) -> Result<bool> {
+    Ok(client
+        .query_one(
+            "SELECT has_schema_privilege($1::name, $2::text, 'USAGE')",
+            &[&pg_role, &schema],
+        )
+        .await
+        .map_err(|e| {
+            Error::internal_err(format!(
+                "Failed to read the role's privileges on schema {schema}: {}",
+                pg_error_message(&e)
+            ))
+        })?
+        .get(0))
+}
+
 /// Postgres only hands an object to a role that could have created it: a table to one with
 /// `CREATE` on its schema, a schema to one with `CREATE` on the database.
 async fn missing_owner_privilege(

backend/windmill-api-workspaces/src/workspaces.rs

@@ -10513,7 +10513,7 @@ async fn edit_workspace(
 pub(crate) async fn archive_workspace_impl(
     db: &DB,
     w_id: &str,
-    username: &str,
+    authed: &ApiAuthed,
     // When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the
     // prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the
     // archive — a later failure can't strand a half-archived dev that's still flagged/locked.
@@ -10627,7 +10627,7 @@ pub(crate) async fn archive_workspace_impl(
         let axum::Json(canceled_jobs) = windmill_api_jobs::cancel_jobs(
             jobs_to_cancel,
             db,
-            username,
+            authed,
             w_id,
             false, // force_cancel
         )
@@ -10687,7 +10687,7 @@ async fn archive_workspace(
     // dev workspace, which only applies to a workspace that is itself a dev (`dev_lock_parent`): a
     // root archived out from under its dev is the pre-existing shape and not this pairing's to police.
     let (schedules_count, canceled_count, deleted_tokens_count) =
-        archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?;
+        archive_workspace_impl(&db, &w_id, &authed, dev_lock_parent.as_deref()).await?;
 
     // Audit log
     let mut tx = db.begin().await?;

backend/windmill-api-workspaces/src/workspaces_extra.rs

@@ -944,7 +944,7 @@ pub(crate) async fn change_workspace_id(
     // Note: schedules were already moved to new workspace, so this will find 0 schedules
     info!("Archiving old workspace");
     let (_schedules_count, canceled_count, _deleted_tokens_count) =
-        archive_workspace_impl(&db, &old_id, &authed.username, None).await?;
+        archive_workspace_impl(&db, &old_id, &authed, None).await?;
 
     // The old id stays live between the commit above and the archive, and a fork copy created for
     // it in that window registers under it. Creation checks the workspace is live under the fork

backend/windmill-api/openapi-deref.json

@@ -34993,7 +34993,7 @@
           },
           "exponential": {
             "type": "object",
-            "description": "Retry with exponential backoff (delay doubles each time)",
+            "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n",
             "properties": {
               "attempts": {
                 "type": "integer",
@@ -35006,7 +35006,7 @@
               "seconds": {
                 "type": "integer",
                 "minimum": 1,
-                "description": "Initial delay in seconds"
+                "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too"
               },
               "random_factor": {
                 "type": "integer",
@@ -47930,7 +47930,7 @@
           },
           "exponential": {
             "type": "object",
-            "description": "Retry with exponential backoff (delay doubles each time)",
+            "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n",
             "properties": {
               "attempts": {
                 "type": "integer",
@@ -47943,7 +47943,7 @@
               "seconds": {
                 "type": "integer",
                 "minimum": 1,
-                "description": "Initial delay in seconds"
+                "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too"
               },
               "random_factor": {
                 "type": "integer",

backend/windmill-api/openapi-deref.yaml

@@ -11725,9 +11725,7 @@ paths:
                                           description: Seconds to wait between retries
                                     exponential:
                                       type: object
-                                      description: >-
-                                        Retry with exponential backoff (delay
-                                        doubles each time)
+                                      description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n'
                                       properties:
                                         attempts:
                                           type: integer
@@ -11738,7 +11736,7 @@ paths:
                                         seconds:
                                           type: integer
                                           minimum: 1
-                                          description: Initial delay in seconds
+                                          description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too
                                         random_factor:
                                           type: integer
                                           minimum: 0
@@ -23272,7 +23270,7 @@ paths:
                           description: Seconds to wait between retries
                     exponential:
                       type: object
-                      description: Retry with exponential backoff (delay doubles each time)
+                      description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n'
                       properties:
                         attempts:
                           type: integer
@@ -23283,7 +23281,7 @@ paths:
                         seconds:
                           type: integer
                           minimum: 1
-                          description: Initial delay in seconds
+                          description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too
                         random_factor:
                           type: integer
                           minimum: 0

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.826.0
+  version: 1.828.0
   title: Windmill API
 
   contact:
@@ -1765,6 +1765,12 @@ paths:
                   type: string
                 cluster:
                   $ref: "#/components/schemas/DatatableRoleCluster"
+                take_over:
+                  type: boolean
+                  description: >
+                    manage a Postgres role of that name that already exists on the cluster,
+                    replacing its password, instead of refusing it. A role with cluster-wide
+                    attributes or memberships is refused either way
       responses:
         "200":
           description: the created role
@@ -1772,6 +1778,8 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/InstanceDatatableRole"
+        "409":
+          description: a Postgres role of that name already exists on the cluster and take_over was not set
 
   /settings/datatable_roles/{id}:
     post:
@@ -12004,6 +12012,7 @@ paths:
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
+        - $ref: "#/components/parameters/RunRetry"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
           in: query
@@ -16008,6 +16017,7 @@ paths:
         - $ref: "#/components/parameters/WorkerTag"
         - $ref: "#/components/parameters/CacheTtl"
         - $ref: "#/components/parameters/NewJobId"
+        - $ref: "#/components/parameters/RunRetry"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
@@ -17312,6 +17322,27 @@ paths:
               schema:
                 type: string
 
+  /w/{workspace}/jobs/worker_is_draining/{id}:
+    get:
+      summary: whether the worker running this job is draining
+      description: |
+        True once the worker running the job has received its shutdown signal. The worker
+        still waits for the job to finish, so a long-running job can poll this to exit early.
+        False when the job is not currently running on a worker.
+      operationId: workerIsDraining
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: whether the worker running this job is draining
+          content:
+            application/json:
+              schema:
+                type: boolean
+
   /w/{workspace}/jobs_u/get_logs/{id}:
     get:
       summary: get job logs
@@ -27822,6 +27853,12 @@ components:
       in: query
       schema:
         type: string
+    RunRetry:
+      name: retry
+      description: JSON-encoded retry policy (same shape as a schedule's retry) re-running the job when it fails. The returned id is the first attempt's; each retry is a job of its own, recorded as a child of the first attempt. Not supported for hub scripts, with invisible_to_owner, or for a script whose preprocessor would run.
+      in: query
+      schema:
+        type: string
     NewJobId:
       name: job_id
       description:
@@ -34525,6 +34562,11 @@ components:
           type: boolean
         cluster:
           $ref: "#/components/schemas/DatatableRoleCluster"
+        in_use:
+          type: boolean
+          description: >-
+            Whether any data table, in any workspace, lets someone connect as this role. Only set
+            in a data table's available_roles.
 
     DatatableRoleTenants:
       type: object
@@ -35880,6 +35922,9 @@ components:
           type: string
         native_mode:
           type: boolean
+        draining:
+          type: boolean
+          description: the worker received its shutdown signal and exits once its current job is done
       required:
         - worker
         - worker_instance

backend/windmill-api/src/apps.rs

@@ -979,7 +979,14 @@ async fn get_raw_app_data(
             // the opaque, sandboxed wrapper (incl. under a cross-origin-isolated
             // / COEP `require-corp` embedder).
             .header("X-Content-Type-Options", "nosniff")
-            .header("Cross-Origin-Resource-Policy", "cross-origin");
+            .header("Cross-Origin-Resource-Policy", "cross-origin")
+            // The secret names one app version, whose bundle is written once at deploy,
+            // so the URL's content never changes. `private`: the URL is a capability, so
+            // shared caches must not keep serving it after the app is deleted.
+            .header(
+                http::header::CACHE_CONTROL,
+                "private, max-age=31536000, immutable",
+            );
         Ok(res.body(body).unwrap())
     } else {
         return Err(Error::NotFound("File not found".to_string()));

backend/windmill-api/src/jobs.rs

@@ -378,6 +378,10 @@ pub fn workspaced_service() -> Router {
             "/wac_approval_urls/{job_id}/{step_key}",
             get(get_wac_approval_urls).layer(cors.clone()),
         )
+        .route(
+            "/worker_is_draining/{id}",
+            get(worker_is_draining).layer(cors.clone()),
+        )
         .route(
             "/result_by_id/{job_id}/{node_id}",
             get(get_result_by_id).layer(cors.clone()),
@@ -613,6 +617,53 @@ async fn get_root_job(
     Ok(Json(res))
 }
 
+/// Whether the worker running this job has received its shutdown signal. A draining worker
+/// never interrupts its job, so a long-running script polls this to exit on its own terms.
+async fn worker_is_draining(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> windmill_common::error::JsonResult<bool> {
+    // A job always may ask about itself. Anyone else must be able to see the job: the
+    // caller's row policies on v2_job decide, narrowed by the token's tag filter.
+    let draining = if authed.job_id == Some(id) {
+        sqlx::query_scalar!(
+            "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker
+            WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
+            id,
+            w_id
+        )
+        .fetch_optional(&db)
+        .await?
+    } else {
+        let tags = get_scope_tags(&authed)
+            .map(|tags| tags.into_iter().map(str::to_string).collect::<Vec<_>>());
+        let mut tx = user_db.begin(&authed).await?;
+        let draining = sqlx::query_scalar!(
+            "SELECT wp.draining FROM v2_job j
+            JOIN v2_job_queue q ON q.id = j.id
+            JOIN worker_ping wp ON wp.worker = q.worker
+            WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL
+            AND ($3::text[] IS NULL OR j.tag = ANY($3))",
+            id,
+            w_id,
+            tags.as_deref()
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+        tx.commit().await?;
+        draining
+    }
+    .unwrap_or(false);
+    windmill_common::feature_usage::log_feature_usage(
+        "worker_draining",
+        "check",
+        if draining { "draining" } else { "not_draining" },
+    );
+    Ok(Json(draining))
+}
+
 async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result<String> {
     let root_job = sqlx::query_scalar!(
             r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#,
@@ -4580,7 +4631,7 @@ async fn cancel_selection(
         let Json(mut w_cancelled) = cancel_jobs(
             ids,
             &db,
-            authed.username.as_str(),
+            &authed,
             workspace_id.as_str(),
             force_cancel,
         )
@@ -8541,6 +8592,11 @@ pub async fn stream_job(
     run_query: RunJobQuery,
     is_get: bool,
 ) -> error::Result<Response> {
+    if run_query.retry.is_some() {
+        return Err(error::Error::BadRequest(
+            "retry is only supported when running a script asynchronously".to_string(),
+        ));
+    }
     let args = if is_get {
         let payload_as_args = run_query.payload_as_args()?;
 
@@ -10483,26 +10539,29 @@ pub async fn run_job_by_hash_inner(
         )
     };
 
+    let job_payload = JobPayload::ScriptHash {
+        hash: ScriptHash(hash),
+        path: path,
+        concurrency_settings,
+        debouncing_settings,
+        cache_ttl,
+        cache_ignore_s3_path,
+        language,
+        dedicated_worker,
+        priority,
+        apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
+            && has_preprocessor.unwrap_or(false),
+        labels,
+        job_token_scopes,
+    };
+    let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?;
+
     let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
     let (uuid, tx) = push(
         &db,
         tx,
         &w_id,
-        JobPayload::ScriptHash {
-            hash: ScriptHash(hash),
-            path: path,
-            concurrency_settings,
-            debouncing_settings,
-            cache_ttl,
-            cache_ignore_s3_path,
-            language,
-            dedicated_worker,
-            priority,
-            apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
-                && has_preprocessor.unwrap_or(false),
-            labels,
-            job_token_scopes,
-        },
+        job_payload,
         push_args,
         authed.display_username(),
         email,

backend/windmill-api/src/mcp/auto_generated_endpoints.rs

@@ -2024,7 +2024,7 @@ You should get the schema of the script or flow before creating the schedule to
                                 },
                                 "exponential": {
                                         "type": "object",
-                                        "description": "Retry with exponential backoff (delay doubles each time)",
+                                        "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n",
                                         "properties": {
                                                 "attempts": {
                                                         "type": "integer",
@@ -2037,7 +2037,7 @@ You should get the schema of the script or flow before creating the schedule to
                                                 "seconds": {
                                                         "type": "integer",
                                                         "minimum": 1,
-                                                        "description": "Initial delay in seconds"
+                                                        "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too"
                                                 },
                                                 "random_factor": {
                                                         "type": "integer",
@@ -2231,7 +2231,7 @@ You should get the schema of the script or flow before updating the schedule to
                                 },
                                 "exponential": {
                                         "type": "object",
-                                        "description": "Retry with exponential backoff (delay doubles each time)",
+                                        "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n",
                                         "properties": {
                                                 "attempts": {
                                                         "type": "integer",
@@ -2244,7 +2244,7 @@ You should get the schema of the script or flow before updating the schedule to
                                                 "seconds": {
                                                         "type": "integer",
                                                         "minimum": 1,
-                                                        "description": "Initial delay in seconds"
+                                                        "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too"
                                                 },
                                                 "random_factor": {
                                                         "type": "integer",

backend/windmill-audit/Cargo.toml

@@ -20,4 +20,7 @@ chrono.workspace = true
 serde_json.workspace = true
 tracing.workspace = true
 windmill-common = { workspace = true, default-features = false }
-lazy_static.workspace = true
+lazy_static.workspace = true
+
+[dev-dependencies]
+tracing-subscriber.workspace = true

backend/windmill-audit/src/audit_oss.rs

@@ -88,6 +88,21 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
     Ok(())
 }
 
+/// AUTHORIZATION: records entries, checks nothing. The caller must have authorized `author`
+/// for the operation on every resource in `w_id` before calling.
+#[cfg(not(feature = "private"))]
+pub async fn audit_log_many<'c, E: sqlx::Executor<'c, Database = Postgres>>(
+    _db: E,
+    _author: &impl AuditAuthorable,
+    _operation: &str,
+    _action_kind: ActionKind,
+    _w_id: &str,
+    _resources: &[String],
+) -> Result<()> {
+    // Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
+    Ok(())
+}
+
 #[cfg(not(feature = "private"))]
 pub async fn list_audit(
     _tx: Transaction<'_, Postgres>,

backend/windmill-common/src/datatable_roles.rs

@@ -331,6 +331,12 @@ pub async fn converge_connect_grants_with(
 /// `CREATE ROLE <name> LOGIN PASSWORD ...; GRANT <name> TO custom_instance_user` on `cluster`. No
 /// privileges beyond that — an admin grants them through SQL or the ACL editor.
 ///
+/// A role of that name that already exists is refused with a 409, unless `take_over`: then its
+/// password is replaced and it is managed from here on like one created here. A role holding
+/// cluster-wide privileges, or inheriting another role's, is refused either way, since every tenant
+/// would get them. Returns whether an existing role was taken over: undoing a failed take-over
+/// must not drop a role that was there before.
+///
 /// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external
 /// cluster is another server: the role is created there before `tx` commits, and callers MUST drop
 /// it again ([`drop_datatable_role`]) if `tx` then fails to commit.
@@ -343,8 +349,10 @@ pub async fn create_datatable_role(
     cluster: DatatableRoleCluster,
     name: &str,
     password: &str,
-) -> Result<()> {
-    crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await
+    take_over: bool,
+) -> Result<bool> {
+    crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password, take_over)
+        .await
 }
 
 /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin

backend/windmill-common/src/datatable_roles_oss.rs

@@ -125,7 +125,8 @@ mod ce {
         _cluster: DatatableRoleCluster,
         _name: &str,
         _password: &str,
-    ) -> Result<()> {
+        _take_over: bool,
+    ) -> Result<bool> {
         Err(unavailable())
     }
 

backend/windmill-common/src/instance_config.rs

@@ -963,6 +963,9 @@ pub struct WorkerGroupConfig {
     pub autoscaling: Option<AutoscalingConfig>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub native_mode: Option<bool>,
+    /// Workers of the group pull no new job while this is set; queued jobs wait.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub paused: Option<bool>,
     /// Object store this group's dependency cache uses instead of the instance one. Same shape
     /// as the instance `object_store_cache_config`.
     #[serde(skip_serializing_if = "Option::is_none")]

backend/windmill-common/src/log_context.rs

@@ -49,6 +49,7 @@ pub struct LogContext {
 
     // Worker / job span (windmill-worker/src/worker.rs)
     pub worker: Option<String>,
+    pub worker_group: Option<String>,
     pub hostname: Option<String>,
     pub tag: Option<String>,
     pub job_id: Option<String>,

backend/windmill-common/src/otel_oss.rs

@@ -94,6 +94,9 @@ pub fn otel_incr_zombie_delete_count(_count: u64) {}
 #[cfg(not(feature = "private"))]
 pub fn otel_set_queue_count(_tag: &str, _count: i64) {}
 
+#[cfg(not(feature = "private"))]
+pub fn otel_set_queue_delay(_tag: &str, _delay_secs: f64) {}
+
 #[cfg(not(feature = "private"))]
 pub fn otel_set_queue_running_count(_tag: &str, _count: i64) {}
 

backend/windmill-common/src/tracing_init.rs

@@ -44,6 +44,11 @@ lazy_static::lazy_static! {
 /// Use `tracing::info!(target: windmill_common::tracing_init::VERBOSE_TARGET, ...)` for logs that should be suppressed in quiet mode.
 pub const VERBOSE_TARGET: &str = "windmill_verbose";
 
+/// Target of audit events, so a log pipeline can route them without matching on content.
+/// It keeps the `windmill` prefix because that is what a `RUST_LOG` of `info` or `debug`
+/// is rewritten to filter on.
+pub const AUDIT_TARGET: &str = "windmill:audit";
+
 /// Prefix used by user scripts to emit a log line as an OTEL tracing event
 /// when `OTEL_JOB_LOGS=true`. Stripped before forwarding to the tracing layer.
 pub const OTEL_PREFIX: &str = "OTEL: ";

backend/windmill-common/src/worker.rs

@@ -477,6 +477,7 @@ lazy_static::lazy_static! {
         env_vars: Default::default(),
         native_mode: false,
         object_store_cache_config: Default::default(),
+        paused: false,
     });
 
     pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
@@ -844,11 +845,14 @@ pub fn make_suspended_pull_query(tags: &[String]) -> String {
 }
 // pub async fn make_suspended
 pub async fn store_suspended_pull_query(wc: &WorkerConfig) {
-    if wc.worker_tags.len() == 0 {
-        tracing::error!("Empty tags in worker tags, skipping");
-        return;
-    }
-    let query = make_suspended_pull_query(&wc.worker_tags);
+    // An empty tag list must clear the query: keeping the previous one would leave a reloaded
+    // worker pulling suspended jobs for tags it no longer listens to.
+    let query = if wc.worker_tags.is_empty() {
+        tracing::error!("Empty tags in worker tags, pulling no suspended jobs");
+        String::new()
+    } else {
+        make_suspended_pull_query(&wc.worker_tags)
+    };
     WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query));
 }
 
@@ -922,6 +926,42 @@ pub async fn store_pull_query(wc: &WorkerConfig) {
     WORKER_PULL_QUERIES_FAIRNESS.store(std::sync::Arc::new(fairness_queries));
 }
 
+const WORKER_GROUP_PAUSED_CACHE_TTL: Duration = Duration::from_secs(10);
+
+lazy_static::lazy_static! {
+    static ref WORKER_GROUP_PAUSED_CACHE: quick_cache::sync::Cache<String, (bool, std::time::Instant)> =
+        quick_cache::sync::Cache::new(1000);
+}
+
+/// Whether the config of `worker_group` is paused, as the server sees it. A worker that reads
+/// its own config knows from `WORKER_CONFIG`; this is for the server deciding on behalf of
+/// workers that do not, and is cached since they ask on every pull.
+pub async fn is_worker_group_paused(db: &DB, worker_group: &str) -> bool {
+    if let Some((paused, at)) = WORKER_GROUP_PAUSED_CACHE.get(worker_group) {
+        if at.elapsed() < WORKER_GROUP_PAUSED_CACHE_TTL {
+            return paused;
+        }
+    }
+    let paused = match sqlx::query_scalar!(
+        "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1",
+        format!("worker__{worker_group}")
+    )
+    .fetch_optional(db)
+    .await
+    {
+        Ok(paused) => paused.flatten().unwrap_or(false),
+        Err(e) => {
+            tracing::error!("Could not read whether worker group {worker_group} is paused: {e:#}");
+            return false;
+        }
+    };
+    WORKER_GROUP_PAUSED_CACHE.insert(
+        worker_group.to_string(),
+        (paused, std::time::Instant::now()),
+    );
+    paused
+}
+
 lazy_static::lazy_static! {
     pub static ref WINDMILL_DIR: String = {
         let dir = std::env::var("WINDMILL_DIR")
@@ -1912,6 +1952,7 @@ pub enum PingType {
     MainLoop,
     Job,
     InitScript,
+    Draining,
 }
 #[derive(Serialize, Deserialize)]
 pub struct Ping {
@@ -1934,6 +1975,8 @@ pub struct Ping {
     pub occupancy_rate_30m: Option<f32>,
     pub job_isolation: Option<String>,
     pub native_mode: Option<bool>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub draining: Option<bool>,
     pub ping_type: PingType,
 }
 pub async fn update_ping_http(
@@ -1986,6 +2029,7 @@ pub async fn update_ping_http(
                 insert_ping.memory,
                 insert_ping.job_isolation,
                 insert_ping.native_mode.unwrap_or(false),
+                insert_ping.draining,
                 db,
             )
             .await?;
@@ -2014,10 +2058,21 @@ pub async fn update_ping_http(
             )
             .await?
         }
+        PingType::Draining => set_worker_draining_query(worker_name, db).await?,
     }
     Ok(())
 }
 
+pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> {
+    sqlx::query!(
+        "UPDATE worker_ping SET draining = true WHERE worker = $1",
+        worker_name
+    )
+    .execute(db)
+    .await?;
+    Ok(())
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct JobCancelled {
     pub canceled_by: String,
@@ -2116,6 +2171,9 @@ pub async fn fetch_raw_script_from_app_query(
 /// `started_at` and `jobs_executed` are the only two columns carried over, being the
 /// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP
 /// lookup has not resolved yet and the predecessor's address is still the best guess.
+///
+/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering
+/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime.
 pub async fn insert_ping_query(
     worker_instance: &str,
     worker_name: &str,
@@ -2129,6 +2187,7 @@ pub async fn insert_ping_query(
     memory: Option<i64>,
     job_isolation: Option<String>,
     native_mode: bool,
+    draining: Option<bool>,
     db: &DB,
 ) -> anyhow::Result<i32> {
     // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and
@@ -2136,8 +2195,8 @@ pub async fn insert_ping_query(
     // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The
     // literal below must stay equal to `external_ip::UNKNOWN_IP`.
     let previous_jobs_executed = sqlx::query_scalar!(
-        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
-        DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL
+        "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)
+        DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)
         RETURNING jobs_executed",
         worker_instance,
         worker_name,
@@ -2151,6 +2210,7 @@ pub async fn insert_ping_query(
         memory,
         job_isolation.as_deref(),
         native_mode,
+        draining,
         )
         .fetch_one(db)
         .await?;
@@ -2470,6 +2530,12 @@ pub async fn load_worker_config(
             .collect();
         priority_tags_sorted.push(PriorityTags { priority: 0, tags: Vec::from_iter(all_tags_set) }); // push the tags that were not listed as high priority with a priority = 0
         priority_tags_sorted.sort_by_key(|elt| Reverse(elt.priority)); // sort by priority DESC
+
+        // The buckets come out of hash maps: without a fixed order two loads of the same
+        // config compare unequal, and a reload reads that as a config change.
+        for bucket in priority_tags_sorted.iter_mut() {
+            bucket.tags.sort();
+        }
     } else {
         // if no priority is used, push all tags with a priority to 0
         priority_tags_sorted.push(PriorityTags { priority: 0, tags: worker_tags.clone() });
@@ -2539,6 +2605,7 @@ pub async fn load_worker_config(
         env_vars: resolved_env_vars,
         native_mode,
         object_store_cache_config: config.object_store_cache_config,
+        paused: config.paused.unwrap_or(false),
     })
 }
 
@@ -2629,6 +2696,7 @@ pub struct WorkerConfigOpt {
     pub env_vars_allowlist: Option<Vec<String>>,
     pub native_mode: Option<bool>,
     pub object_store_cache_config: Option<serde_json::Value>,
+    pub paused: Option<bool>,
 }
 
 impl Default for WorkerConfigOpt {
@@ -2648,6 +2716,7 @@ impl Default for WorkerConfigOpt {
             env_vars_allowlist: Default::default(),
             native_mode: Default::default(),
             object_store_cache_config: Default::default(),
+            paused: Default::default(),
         }
     }
 }
@@ -2670,12 +2739,15 @@ pub struct WorkerConfig {
     /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate
     /// that parses it, and comparing the raw value is what tells a reload the store changed.
     pub object_store_cache_config: Option<serde_json::Value>,
+    /// The group pulls no job from the queue: its workers finish what they hold, keep pinging
+    /// and leave queued jobs for when it is resumed.
+    pub paused: bool,
 }
 
 impl std::fmt::Debug for WorkerConfig {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}",
-        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.native_mode,
+        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}",
+        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.native_mode, self.paused,
         // holds bucket credentials
         self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None"))
     }

backend/windmill-queue/src/jobs.rs

@@ -359,6 +359,22 @@ ORDER BY depth, id
     .filter_map(|r| r.id.clone())
     .collect_vec();
 
+    // A native retry is the child of its first attempt, which has completed by the time the
+    // retry is queued, so the walk above stops before reaching it.
+    let tree = [&[job.id][..], &jobs_to_cancel[..]].concat();
+    let pending_retries = sqlx::query_scalar!(
+        "SELECT q.id FROM v2_job_queue q
+            JOIN native_retry_attempt n ON n.job_id = q.id
+            JOIN v2_job r ON r.id = q.id
+            JOIN v2_job a ON a.id = r.parent_job
+            WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)",
+        tree.as_slice(),
+        w_id
+    )
+    .fetch_all(&mut *tx)
+    .await?;
+    jobs_to_cancel.extend(pending_retries);
+
     jobs_to_cancel.reverse();
     if !jobs_to_cancel.is_empty() {
         tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len());
@@ -1391,7 +1407,18 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
             .then(|| completed_job.parent_job)
             .flatten()
     } else if !success && !skipped && retry_pending {
-        Some(completed_job.parent_job.unwrap_or(completed_job.id))
+        let is_retry_attempt = sqlx::query_scalar!(
+            "SELECT attempt FROM native_retry_attempt WHERE job_id = $1",
+            completed_job.id,
+        )
+        .fetch_optional(db)
+        .await
+        .unwrap_or_else(|e| {
+            tracing::error!("reading the retry marker of {}: {e:#}", completed_job.id);
+            None
+        })
+        .is_some();
+        Some(native_retry_root(completed_job, is_retry_attempt))
     } else {
         None
     };
@@ -2194,18 +2221,17 @@ async fn restart_job_if_perpetual_inner(
 /// resolution too; `ON CONFLICT DO NOTHING` keeps a human's note intact.
 ///
 /// Membership of the chain must be *proven* per row, never inferred from `parent_job`
-/// alone: `root` is `job.parent_job.unwrap_or(job.id)`, so for a job launched with an
-/// explicit `parent_job` (WAC inline children, SDK-launched children) `root` is the
-/// *calling* job, and its other failed children are unrelated. Resolving those would hide
-/// exactly the failures this feature exists to surface, so each row must be either
+/// alone: the success-side caller passes the succeeding job's `parent_job`, which for a
+/// job launched with an explicit parent (WAC inline children, SDK-launched children) is
+/// the *calling* job, and its other failed children are unrelated. Resolving those would
+/// hide exactly the failures this feature exists to surface, so each row must be either
 /// - a job carrying a `native_retry_attempt` marker under `root` (provably an attempt), or
-/// - `root` itself as the original attempt, which is only provable when `root` is
-///   parentless and unmarked.
+/// - `root` itself as the original attempt: unmarked, and proven a chain root by the
+///   marked success under it, since only a chain root is ever a marked job's parent
+///   (see [`native_retry_root`]).
 ///
 /// Both arms also require the same `runnable_id` as the succeeding attempt, since every
-/// attempt in a chain runs the same runnable. The deliberate cost is a miss, not an
-/// over-reach: when the original attempt had a parent it is an unmarked sibling
-/// indistinguishable from any other child of the caller, so it stays red.
+/// attempt in a chain runs the same runnable.
 ///
 /// Nothing is trusted of the caller (this writes through a privileged pool): the gate is
 /// "some marked attempt under `root` running `runnable_id` has succeeded", evaluated in
@@ -2249,7 +2275,7 @@ pub async fn resolve_retry_chain_if_succeeded(
                     AND (
                         (j.parent_job = $1
                             AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))
-                        OR (c.id = $1 AND j.parent_job IS NULL
+                        OR (c.id = $1
                             AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))
                     )
             ON CONFLICT (job_id) DO NOTHING",
@@ -2303,6 +2329,18 @@ async fn eval_retry_if(
     false
 }
 
+/// The first attempt of `job`'s native retry chain, which every later attempt is pushed
+/// with as `parent_job` and whose id seeds theirs. A first attempt may have a parent of its
+/// own (a script dispatched from another job), so only a marked attempt's parent is the root:
+/// rooting at that outer parent would merge the chains of its retried children.
+fn native_retry_root(job: &MiniCompletedJob, is_retry_attempt: bool) -> Uuid {
+    if is_retry_attempt {
+        job.parent_job.unwrap_or(job.id)
+    } else {
+        job.id
+    }
+}
+
 /// Native script retry. When a failed `Script` or `Script_Hub` job carries a retry
 /// policy (via `runnable_settings_handle`) and has attempts left, enqueue a fresh
 /// attempt of the same script after the policy's backoff delay — instead of having
@@ -2346,6 +2384,18 @@ pub async fn maybe_enqueue_native_script_retry(
     if !policy.has_attempts() {
         return Ok(false);
     }
+    // Cancelling a job that has a parent (every retry attempt does) only flags its queue row,
+    // and the worker that pulls it completes it without a `canceled_by`: read the flag here.
+    let soft_canceled = sqlx::query_scalar!(
+        "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1",
+        job.id,
+    )
+    .fetch_optional(db)
+    .await?
+    .unwrap_or(false);
+    if soft_canceled {
+        return Ok(false);
+    }
 
     // Attempt counter for this job: its `native_retry_attempt` marker, or 0 for
     // the first (un-marked) attempt. The marker is persistent (unlike the queue
@@ -2358,7 +2408,7 @@ pub async fn maybe_enqueue_native_script_retry(
     .fetch_optional(db)
     .await?
     .unwrap_or(0) as u32;
-    let root = job.parent_job.unwrap_or(job.id);
+    let root = native_retry_root(job, prev_attempts > 0);
     // Scheduled chains keep the schedule trigger so the terminal attempt drives
     // the schedule completion handlers (on_failure/on_success); `parent_job`
     // keeps every retry out of the per-occurrence handler counting queries.

backend/windmill-queue/tests/concurrency_counter_test.rs

@@ -33,32 +33,19 @@ mod concurrency {
         .unwrap_or(0)
     }
 
-    /// Stress: N concurrent admit-attempts on a pre-existing key with limit
-    /// L < N. Asserts admitted count is exactly L (never over-admits) AND
-    /// that the row's job_uuids reflects exactly L entries.
+    /// Stress: N concurrent admit-attempts on a key with limit L < N. Asserts
+    /// admitted count is exactly L (never over-admits) AND that the row's
+    /// job_uuids reflects exactly L entries.
     ///
-    /// The row is pre-seeded with an empty `job_uuids` because there is a
-    /// known preexisting race when the `concurrency_counter` row does not
-    /// exist yet: `SELECT ... FOR UPDATE` on a missing row locks nothing,
-    /// so all concurrent transactions see `running=0` and all admit via
-    /// `INSERT ... ON CONFLICT DO UPDATE`. That race is inherited from
-    /// the legacy code path and is out of scope for this PR. This test
-    /// validates the property we *do* preserve: no over-admit once the
-    /// row exists.
+    /// The `concurrency_counter` row is deliberately absent at the start: the
+    /// monitor deletes empty rows while jobs for the key are still queued, so
+    /// the admit path has to serialize without a pre-existing row to lock.
     #[sqlx::test(migrations = "../migrations")]
     async fn stress_never_over_admits(db: Pool<Postgres>) {
         let key = "stress-never-over-admits".to_string();
         let limit: i32 = 5;
         let workers: usize = 50;
 
-        sqlx::query!(
-            "INSERT INTO concurrency_counter(concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb)",
-            key,
-        )
-        .execute(&db)
-        .await
-        .expect("seed empty counter row");
-
         let mut tasks = Vec::with_capacity(workers);
         for _ in 0..workers {
             let db = db.clone();

backend/windmill-queue/tests/native_retry_test.rs

@@ -561,4 +561,123 @@ mod native_retry {
                 .unwrap();
         assert_eq!(status, "failure", "resolving must not change job status");
     }
+
+    // A script dispatched from another job keeps that job as its parent, so its chain is
+    // rooted at the first attempt itself: sibling dispatches must not share retry ids, and
+    // a parented first attempt is still resolved once its retry succeeds.
+    #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))]
+    async fn parented_first_attempts_root_their_own_chains(
+        db: Pool<Postgres>,
+    ) -> anyhow::Result<()> {
+        let retry = Retry {
+            constant: ConstantDelay { attempts: 1, seconds: 0 },
+            exponential: Default::default(),
+            retry_if: None,
+        };
+        let handle = insert_rs(
+            RunnableSettings {
+                debouncing_settings: None,
+                concurrency_settings: None,
+                retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?,
+            },
+            &db,
+        )
+        .await?;
+        let dispatcher = Uuid::new_v4();
+        let (a, b) = (Uuid::new_v4(), Uuid::new_v4());
+        for first in [a, b] {
+            let attempt = mini(first, Some(dispatcher), handle);
+            assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?);
+        }
+        assert!(
+            retry_by_attempt(&db, a, 1).await.is_some(),
+            "a's retry hangs off a"
+        );
+        assert!(
+            retry_by_attempt(&db, b, 1).await.is_some(),
+            "b's retry is not mistaken for a's"
+        );
+
+        let (c, c_retry, c_sibling) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4());
+        seed_job(&db, c, Some(dispatcher), false, "failure").await;
+        seed_job(&db, c_sibling, Some(dispatcher), false, "failure").await;
+        seed_job(&db, c_retry, Some(c), true, "success").await;
+        windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, c, WS, None).await?;
+        windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, dispatcher, WS, None).await?;
+        let resolved: Vec<Uuid> = sqlx::query_scalar("SELECT job_id FROM job_resolution")
+            .fetch_all(&db)
+            .await?;
+        assert_eq!(
+            resolved,
+            vec![c],
+            "only the parented first attempt resolves, not its sibling"
+        );
+        Ok(())
+    }
+
+    // Cancelling the job that dispatched a script reaches that script's pending retry, which
+    // hangs off a first attempt that has already completed.
+    #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))]
+    async fn cancelling_the_caller_cancels_a_pending_retry(
+        db: Pool<Postgres>,
+    ) -> anyhow::Result<()> {
+        let retry = Retry {
+            constant: ConstantDelay { attempts: 2, seconds: 60 },
+            exponential: Default::default(),
+            retry_if: None,
+        };
+        let handle = insert_rs(
+            RunnableSettings {
+                debouncing_settings: None,
+                concurrency_settings: None,
+                retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?,
+            },
+            &db,
+        )
+        .await?;
+        let (caller, first) = (Uuid::new_v4(), Uuid::new_v4());
+        sqlx::query(
+            "INSERT INTO v2_job (id, workspace_id, kind, runnable_path) VALUES ($1, $2, 'script', $3)",
+        )
+        .bind(caller)
+        .bind(WS)
+        .bind(SCRIPT)
+        .execute(&db)
+        .await?;
+        sqlx::query(
+            "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running) VALUES ($1, $2, now(), true)",
+        )
+        .bind(caller)
+        .bind(WS)
+        .execute(&db)
+        .await?;
+        seed_job(&db, first, Some(caller), false, "failure").await;
+        let attempt = mini(first, Some(caller), handle);
+        assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?);
+        let (retry_id, ..) = retry_by_attempt(&db, first, 1)
+            .await
+            .expect("retry is queued");
+
+        let tx = db.begin().await?;
+        let (tx, _) =
+            windmill_queue::jobs::cancel_job("test-user", None, caller, WS, tx, &db, false, false)
+                .await?;
+        tx.commit().await?;
+
+        let canceled_by: Option<String> =
+            sqlx::query_scalar("SELECT canceled_by FROM v2_job_queue WHERE id = $1")
+                .bind(retry_id)
+                .fetch_one(&db)
+                .await?;
+        assert_eq!(canceled_by.as_deref(), Some("test-user"));
+
+        // The worker that pulls the canceled retry completes it without a `canceled_by`.
+        let (_, _, _, _, retry_handle) = retry_by_attempt(&db, first, 1).await.unwrap();
+        let canceled_retry = mini(retry_id, Some(first), retry_handle);
+        assert!(
+            !maybe_enqueue_native_script_retry(&db, &canceled_retry, &None, &no_result).await?,
+            "a canceled retry does not enqueue the next attempt"
+        );
+        Ok(())
+    }
 }

backend/windmill-runtime-nativets/src/lib.rs

@@ -929,8 +929,8 @@ pub async fn eval_fetch_timeout(
                     for line in logged.as_deref().unwrap_or(&log).lines() {
                         tracing::info!(
                             target: "windmill:job_log",
-                            job_id = ?job_id,
-                            workspace_id = ?w_id_for_tracing,
+                            job_id = %job_id,
+                            workspace_id = %w_id_for_tracing,
                             "{line}"
                         );
                         if *OTEL_JOB_LOGS {

backend/windmill-trigger/src/global_handler.rs

@@ -342,7 +342,7 @@ pub async fn cancel_suspended_trigger_jobs(
         let cancelled_jobs = cancel_jobs(
             jobs_to_cancel,
             &db,
-            authed.username.as_str(),
+            &authed,
             w_id.as_str(),
             true,
         )

backend/windmill-types/src/flows.rs

@@ -1140,7 +1140,7 @@ pub enum FlowModuleValue {
         #[serde(default, skip_serializing_if = "HashMap::is_empty")]
         tool_inputs: HashMap<String, HashMap<String, InputTransform>>,
     },
-    /// One call to a decision model (TypeSafe's Jev) answering the typed `questions` of its
+    /// One call to a decision model (such as TypeSafe's Jev) answering the typed `questions` of its
     /// `input_transforms` about a `state`. A flow branches on the answers with a `BranchOne`.
     AIDecision {
         input_transforms: HashMap<String, InputTransform>,

backend/windmill-worker-volumes/src/lib.rs

@@ -54,11 +54,45 @@ pub struct VolumeMount {
     pub target: String,
 }
 
+/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while
+/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A
+/// write within the filesystem's timestamp granularity of the hand-off is not seen.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct FileStamp {
+    size: u64,
+    changed: Option<(i64, i64)>,
+}
+
+impl FileStamp {
+    /// Whether the file is as it was at `earlier`. An unreadable change time never matches.
+    pub fn unchanged_since(&self, earlier: &FileStamp) -> bool {
+        self.changed.is_some() && self == earlier
+    }
+
+    pub fn of(meta: &std::fs::Metadata) -> Self {
+        // ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it.
+        #[cfg(unix)]
+        let changed = {
+            use std::os::unix::fs::MetadataExt;
+            Some((meta.ctime(), meta.ctime_nsec()))
+        };
+        #[cfg(not(unix))]
+        let changed = meta
+            .modified()
+            .ok()
+            .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
+            .map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64));
+        Self { size: meta.len(), changed }
+    }
+}
+
 pub struct VolumeState {
     pub mount: VolumeMount,
     pub local_dir: PathBuf,
     pub manifest: HashMap<String, FileEntry>,
     pub symlinks: HashMap<String, String>,
+    /// Stamp of each pulled file as the job received it, keyed like `manifest`.
+    pub stamps: HashMap<String, FileStamp>,
 }
 
 pub struct DownloadStats {

backend/windmill-worker/nsjail/build.csharp.config.proto

@@ -0,0 +1,128 @@
+name: "csharp restore/build"
+
+mode: ONCE
+hostname: "csharp"
+log_level: ERROR
+time_limit: {TIMEOUT}
+
+disable_rl: true
+
+cwd: "{JOB_DIR}"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+skip_setsid: true
+keep_caps: false
+keep_env: true
+mount_proc: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+    is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+    is_bind: true
+}
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+    is_bind: true
+}
+
+mount {
+    src: "{DOTNET_ROOT}"
+    dst: "{DOTNET_ROOT}"
+    is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+# Container runtimes bind exactly these 3 files as separate submounts over
+# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable.
+# Load-bearing -- do not remove as redundant with the /etc bind above.
+mount {
+    src: "/etc/resolv.conf"
+    dst: "/etc/resolv.conf"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hosts"
+    dst: "/etc/hosts"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hostname"
+    dst: "/etc/hostname"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/dev/null"
+    dst: "/dev/null"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
+mount {
+    src: "/dev/random"
+    dst: "/dev/random"
+    is_bind: true
+}
+
+{TMP_MOUNT_BLOCK}
+
+# project files, nuget.config, and the published output dir are all here; cwd of dotnet
+mount {
+    src: "{JOB_DIR}"
+    dst: "{JOB_DIR}"
+    is_bind: true
+    rw: true
+}
+
+# DOTNET_CLI_HOME + NUGET_PACKAGES (nuget/). Writable, like the install/run jails'
+# shared caches.
+mount {
+    src: "{CSHARP_CACHE_DIR}"
+    dst: "{CSHARP_CACHE_DIR}"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "{TRACING_PROXY_CA_CERT_PATH}"
+    dst: "{TRACING_PROXY_CA_CERT_PATH}"
+    is_bind: true
+    mandatory: false
+}
+
+envar: "HOME=/tmp"
+
+#{DEV}

backend/windmill-worker/nsjail/lock.php.config.proto

@@ -0,0 +1,124 @@
+name: "php composer install"
+
+mode: ONCE
+hostname: "php"
+log_level: ERROR
+time_limit: {TIMEOUT}
+
+disable_rl: true
+
+cwd: "{JOB_DIR}"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+skip_setsid: true
+keep_caps: false
+keep_env: true
+mount_proc: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+# Container runtimes bind exactly these 3 files as separate submounts over
+# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable.
+# Load-bearing -- do not remove as redundant with the /etc bind above.
+mount {
+    src: "/etc/resolv.conf"
+    dst: "/etc/resolv.conf"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hosts"
+    dst: "/etc/hosts"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hostname"
+    dst: "/etc/hostname"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+	src: "/dev/null"
+	dst: "/dev/null"
+	is_bind: true
+	rw: true
+}
+
+mount {
+	src: "/dev/urandom"
+	dst: "/dev/urandom"
+	is_bind: true
+}
+
+mount {
+	src: "/dev/random"
+	dst: "/dev/random"
+	is_bind: true
+}
+
+{TMP_MOUNT_BLOCK}
+
+# composer.json/composer.lock live here and vendor/ is written here; cwd of composer
+mount {
+    src: "{JOB_DIR}"
+    dst: "{JOB_DIR}"
+    is_bind: true
+    rw: true
+}
+
+# COMPOSER_HOME: package download cache. Writable, like the install/run jails'
+# shared caches.
+mount {
+    src: "{COMPOSER_CACHE_DIR}"
+    dst: "{COMPOSER_CACHE_DIR}"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "{TRACING_PROXY_CA_CERT_PATH}"
+    dst: "{TRACING_PROXY_CA_CERT_PATH}"
+    is_bind: true
+    mandatory: false
+}
+
+{CA_MOUNT}
+
+envar: "HOME=/tmp"
+
+#{DEV}

backend/windmill-worker/nsjail/lock.py.config.proto

@@ -0,0 +1,133 @@
+name: "python lock compile"
+
+mode: ONCE
+hostname: "python"
+log_level: ERROR
+time_limit: {TIMEOUT}
+
+# Dependency resolution may invoke the PEP 517 build backend of an sdist whose
+# metadata is not static, which runs arbitrary package code (e.g. setup.py).
+# That build, and uv's bytecode compile pool, size thread/fd pools to the host's
+# CPU count, so these mirror download.py.config.proto rather than the run sandbox.
+rlimit_as: 8192
+rlimit_cpu: 1000
+rlimit_fsize: 1024
+rlimit_nofile: 10000
+
+envar: "HOME=/tmp"
+envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
+
+cwd: "{JOB_DIR}"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+skip_setsid: true
+keep_caps: true
+keep_env: true
+mount_proc: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+	is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+	is_bind: true
+}
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+	is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+	is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+# Container runtimes bind exactly these 3 files as separate submounts over
+# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable.
+# Load-bearing -- do not remove as redundant with the /etc bind above.
+mount {
+    src: "/etc/resolv.conf"
+    dst: "/etc/resolv.conf"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hosts"
+    dst: "/etc/hosts"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/etc/hostname"
+    dst: "/etc/hostname"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+	src: "/dev/null"
+	dst: "/dev/null"
+	is_bind: true
+	rw: true
+}
+
+mount {
+	src: "/dev/urandom"
+	dst: "/dev/urandom"
+	is_bind: true
+}
+
+{TMP_MOUNT_BLOCK}
+
+# requirements.in -> requirements.txt, cwd of uv
+mount {
+    src: "{JOB_DIR}"
+    dst: "{JOB_DIR}"
+    is_bind: true
+    rw: true
+}
+
+# shared uv cache: resolution metadata + where sdist build backends run
+# (builds-v0/). Writable, as the install jail's target already is; a poisoned
+# shared cache is the known residual, not a new one.
+mount {
+    src: "{UV_CACHE_DIR}"
+    dst: "{UV_CACHE_DIR}"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    src: "{PY_INSTALL_DIR}"
+    dst: "{PY_INSTALL_DIR}"
+    is_bind: true
+}
+
+{INDEX_CERT_MOUNT}
+
+mount {
+    src: "{TRACING_PROXY_CA_CERT_PATH}"
+    dst: "{TRACING_PROXY_CA_CERT_PATH}"
+    is_bind: true
+    mandatory: false
+}
+
+#{DEV}

backend/windmill-worker/src/ai_executor.rs

@@ -28,7 +28,7 @@ use windmill_mcp::McpClient;
 use crate::ai::tools::McpClientStub as McpClient;
 use windmill_ai::{
     ai_providers::AIProvider,
-    decision::{decision_inputs, run_systemone, AIDecisionArgs},
+    decision::{decision_inputs, run_decision, AIDecisionArgs},
     image_handler::upload_image_to_s3,
     model_context::model_context_window,
     providers::{
@@ -2449,9 +2449,9 @@ async fn handle_ai_decision(
     worker_name: &str,
 ) -> error::Result<Box<RawValue>> {
     let args = serde_json::from_str::<AIDecisionArgs>(&serde_json::to_string(local_args)?)?;
-    if !args.provider.kind.is_decision_provider() {
+    if !args.provider.kind.runs_decisions() {
         return Err(Error::BadRequest(format!(
-            "An AI decision runs on a TypeSafe or Cloudflare resource, not {:?}",
+            "An AI decision runs on a TypeSafe, Cloudflare or OpenAI resource, not {:?}",
             args.provider.kind
         )));
     }
@@ -2467,7 +2467,7 @@ async fn handle_ai_decision(
         conn,
         mem_peak,
         canceled_by,
-        run_systemone(
+        run_decision(
             &credentials,
             args.provider.get_model(),
             state,

backend/windmill-worker/src/common.rs

@@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script(
                         wm_memory_usage: None,
                         job_isolation: None,
                         native_mode: None,
+                        draining: None,
                         ping_type: PingType::InitScript,
                     },
                 )

backend/windmill-worker/src/csharp_executor.rs

@@ -33,7 +33,7 @@ use crate::{
     get_proxy_envs_for_lang,
     handle_child::handle_child,
     is_sandboxing_enabled, read_ee_registry_with_workspace_override, CSHARP_CACHE_DIR,
-    DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV,
+    DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, PROXY_ENVS,
     TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
 };
 #[cfg(feature = "csharp")]
@@ -47,6 +47,9 @@ use crate::SYSTEM_ROOT;
 
 #[cfg(feature = "csharp")]
 const NSJAIL_CONFIG_RUN_CSHARP_CONTENT: &str = include_str!("../nsjail/run.csharp.config.proto");
+#[cfg(feature = "csharp")]
+const NSJAIL_CONFIG_BUILD_CSHARP_CONTENT: &str =
+    include_str!("../nsjail/build.csharp.config.proto");
 
 #[cfg(feature = "csharp")]
 #[cfg(windows)]
@@ -124,6 +127,9 @@ pub async fn generate_nuget_lockfile(
 
     gen_cs_proj(code, job_dir, reqs, lines_to_remove)?;
 
+    // `dotnet restore` resolves and downloads packages but does not import their
+    // MSBuild `build/*.targets`, so it runs no package code — only `dotnet publish`
+    // (jailed below) does — so it runs unsandboxed.
     let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str());
     gen_lockfile_cmd
         .current_dir(job_dir)
@@ -384,31 +390,82 @@ async fn build_cs_proj(
         }
     }
 
-    let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str());
-    build_cs_cmd
-        .current_dir(job_dir)
-        .env_clear()
-        .env("PATH", PATH_ENV.as_str())
-        .env("BASE_INTERNAL_URL", base_internal_url)
-        .env("HOME", HOME_ENV.as_str())
-        .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)
-        .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR))
-        .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true")
-        .env("DOTNET_NOLOGO", "true")
-        .env("MSBUILDDISABLENODEREUSE", "1")
-        .env("DOTNET_ROOT", DOTNET_ROOT.as_str())
-        .args(vec![
-            "publish",
-            "--configuration",
-            "Release",
-            "-o",
-            &format!("{job_dir}/out"),
-            "--no-self-contained",
-            "-p:PublishSingleFile=true",
-            "-p:IncludeNativeLibrariesForSelfExtract=true",
-        ])
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped());
+    let out_dir = format!("{job_dir}/out");
+    let publish_args = vec![
+        "publish",
+        "--configuration",
+        "Release",
+        "-o",
+        out_dir.as_str(),
+        "--no-self-contained",
+        "-p:PublishSingleFile=true",
+        "-p:IncludeNativeLibrariesForSelfExtract=true",
+    ];
+    // `dotnet publish` imports and runs the MSBuild targets/tasks of restored NuGet
+    // packages (a package's build/*.targets execute at build, not at restore), i.e.
+    // arbitrary package code. Confine it under nsjail when sandboxing is on, the same
+    // way the run step is; HOME is set to /tmp by the config, keep_env passes the
+    // explicit dotnet/proxy vars, and the job dir + shared nuget cache are writable.
+    // Without sandboxing, publish runs directly with no isolation.
+    // mut is required by the #[cfg(windows)] env block below; unused on unix.
+    #[allow(unused_mut)]
+    let mut build_cs_cmd = if is_sandboxing_enabled() {
+        std::fs::create_dir_all(&*CSHARP_CACHE_DIR)?;
+        let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await;
+        write_file(
+            job_dir,
+            "build.config.proto",
+            NSJAIL_CONFIG_BUILD_CSHARP_CONTENT
+                .replace("{TIMEOUT}", &nsjail_timeout)
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CSHARP_CACHE_DIR}", &*CSHARP_CACHE_DIR)
+                .replace("{DOTNET_ROOT}", DOTNET_ROOT.as_str())
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
+                .replace("#{DEV}", DEV_CONF_NSJAIL)
+                .replace(
+                    "{TMP_MOUNT_BLOCK}",
+                    &resolve_nsjail_tmp_mount_block(job_dir).await,
+                )
+                .as_str(),
+        )?;
+        let mut nsjail_args = vec!["--config", "build.config.proto", "--", DOTNET_PATH.as_str()];
+        nsjail_args.extend(publish_args.iter().copied());
+        let mut cmd = Command::new(NSJAIL_PATH.as_str());
+        cmd.current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)
+            .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR))
+            .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true")
+            .env("DOTNET_NOLOGO", "true")
+            .env("MSBUILDDISABLENODEREUSE", "1")
+            .env("DOTNET_ROOT", DOTNET_ROOT.as_str())
+            .envs(PROXY_ENVS.clone())
+            .envs(&*crate::worker::WHITELIST_ENVS)
+            .args(nsjail_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        cmd
+    } else {
+        let mut cmd = Command::new(DOTNET_PATH.as_str());
+        cmd.current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("HOME", HOME_ENV.as_str())
+            .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR)
+            .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR))
+            .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true")
+            .env("DOTNET_NOLOGO", "true")
+            .env("MSBUILDDISABLENODEREUSE", "1")
+            .env("DOTNET_ROOT", DOTNET_ROOT.as_str())
+            .args(publish_args.clone())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        cmd
+    };
 
     #[cfg(windows)]
     build_cs_cmd
@@ -433,14 +490,20 @@ async fn build_cs_proj(
                 .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
         );
 
-    let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str(), true).await?;
+    let build_cs_executable = if is_sandboxing_enabled() {
+        NSJAIL_PATH.as_str()
+    } else {
+        DOTNET_PATH.as_str()
+    };
+    let build_cs_process = start_child_process(build_cs_cmd, build_cs_executable, true).await?;
     handle_child(
         job_id,
         conn,
         mem_peak,
         canceled_by,
         build_cs_process,
-        false,
+        // jailed when sandboxing is on, so mem sampling finds the real process
+        is_sandboxing_enabled(),
         worker_name,
         w_id,
         "dotnet publish",

backend/windmill-worker/src/handle_child.rs

@@ -210,6 +210,53 @@ pub async fn handle_child(
         }
     }
 
+    impl KillReason {
+        /// Reports a kill once the process is gone: a span event on the enclosing span and,
+        /// through the logs bridge, a log record carrying the job's log context.
+        fn emit_canceled_event(
+            &self,
+            job_id: &Uuid,
+            timeout_reason: &str,
+            signal: &'static str,
+            detected_at: chrono::DateTime<chrono::Utc>,
+        ) {
+            // `force` is only known for a soft cancel: a job completed under the worker
+            // may have been force canceled or completed by anything else server-side.
+            let (kill_reason, canceled_by, canceled_reason, force) = match self {
+                KillReason::TooManyLogs => return,
+                KillReason::Timeout { .. } => {
+                    ("timeout", Some("timeout"), Some(timeout_reason), None)
+                }
+                KillReason::Cancelled(by) => (
+                    "cancelled",
+                    by.as_ref().and_then(|x| x.username.as_deref()),
+                    by.as_ref().and_then(|x| x.reason.as_deref()),
+                    Some(false),
+                ),
+                KillReason::AlreadyCompleted => ("already_completed", None, None, None),
+            };
+            let exited_at = chrono::Utc::now();
+            let root_job = windmill_common::log_context::current_log_context()
+                .and_then(|c| c.root_job.clone());
+            let rfc3339 = |t: chrono::DateTime<chrono::Utc>| {
+                t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)
+            };
+            tracing::info!(
+                job_id = %job_id,
+                root_job,
+                kill_reason,
+                canceled_by,
+                canceled_reason,
+                force,
+                kill_signal = signal,
+                cancel_detected_at = %rfc3339(detected_at),
+                process_exited_at = %rfc3339(exited_at),
+                kill_duration_ms = (exited_at - detected_at).num_milliseconds(),
+                "job.canceled"
+            );
+        }
+    }
+
     let (timeout_duration, timeout_warn_msg, timeout_source) =
         resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await;
     if let Some(msg) = timeout_warn_msg {
@@ -232,6 +279,7 @@ pub async fn handle_child(
                 UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
             },
         };
+        let detected_at = chrono::Utc::now();
         tx.send(()).expect("rx should never be dropped");
         drop(tx);
 
@@ -268,59 +316,78 @@ pub async fn handle_child(
             }
         };
 
-        #[allow(unused_variables)]
-        if let Some(id) = child.id() {
-            if *MAX_WAIT_FOR_SIGINT > 0 {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
+        // Resolves once the process is reaped, to the last signal it was sent, or to `None`
+        // when the kill could not be confirmed. SIGINT and SIGTERM are only sent where the
+        // sends below are compiled in: elsewhere an exit during their wait is unsignaled.
+        let unix_signal = |s: &'static str| {
+            if cfg!(any(target_os = "linux", target_os = "macos")) { s } else { "none" }
+        };
+        let kill = async {
+            #[allow(unused_variables)]
+            if let Some(id) = child.id() {
+                if *MAX_WAIT_FOR_SIGINT > 0 {
+                    #[cfg(any(target_os = "linux", target_os = "macos"))]
+                    signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
 
-                for _ in 0..*MAX_WAIT_FOR_SIGINT {
+                    for _ in 0..*MAX_WAIT_FOR_SIGINT {
+                        if child.try_wait().is_ok_and(|x| x.is_some()) {
+                            break;
+                        }
+                        sleep(Duration::from_secs(1)).await;
+                    }
                     if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
+                        set_reason.await;
+                        return Ok(Some(unix_signal("SIGINT")));
                     }
-                    sleep(Duration::from_secs(1)).await;
                 }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
+                if sigterm {
+                    #[cfg(any(target_os = "linux", target_os = "macos"))]
+                    signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
+
+                    for _ in 0..*MAX_WAIT_FOR_SIGTERM {
+                        if child.try_wait().is_ok_and(|x| x.is_some()) {
+                            break;
+                        }
+                        sleep(Duration::from_secs(1)).await;
+                    }
+                    if child.try_wait().is_ok_and(|x| x.is_some()) {
+                        set_reason.await;
+                        return Ok(Some(unix_signal("SIGTERM")));
+                    }
                 }
             }
-            if sigterm {
-                #[cfg(any(target_os = "linux", target_os = "macos"))]
-                signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
-
-                for _ in 0..*MAX_WAIT_FOR_SIGTERM {
-                    if child.try_wait().is_ok_and(|x| x.is_some()) {
-                        break;
+            #[cfg(windows)]
+            {
+                let pid_to_kill = child.id();
+                let killed = match kill_process_tree(pid_to_kill).await {
+                    Ok(_) => {
+                        tracing::debug!(
+                            "successfully killed process tree with PID: {:?}",
+                            pid_to_kill
+                        );
+                        true
                     }
-                    sleep(Duration::from_secs(1)).await;
-                }
-                if child.try_wait().is_ok_and(|x| x.is_some()) {
-                    set_reason.await;
-                    return Ok(Err(kill_reason));
-                }
+                    Err(e) => {
+                        tracing::error!("failed to kill process tree: {:?}", e);
+                        false
+                    }
+                };
+                set_reason.await;
+                return Ok(killed.then_some("taskkill"));
             }
-        }
-        #[cfg(windows)]
-        {
-            let pid_to_kill = child.id();
-            match kill_process_tree(pid_to_kill).await {
-                Ok(_) => tracing::debug!(
-                    "successfully killed process tree with PID: {:?}",
-                    pid_to_kill
-                ),
-                Err(e) => tracing::error!("failed to kill process tree: {:?}", e),
-            };
-            set_reason.await;
-            return Ok(Err(kill_reason));
-        }
 
-        #[cfg(unix)]
-        {
-            /* send SIGKILL and reap child process */
-            let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await;
-            kill.map(|()| Err(kill_reason))
+            #[cfg(unix)]
+            {
+                /* send SIGKILL and reap child process */
+                let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await;
+                kill.map(|()| Some("SIGKILL"))
+            }
+        };
+        let signal: io::Result<Option<&'static str>> = kill.await;
+        if let Some(signal) = signal? {
+            kill_reason.emit_canceled_event(&job_id, &timeout_reason, signal, detected_at);
         }
+        Ok(Err(kill_reason))
     };
 
     let mut stream_result = Vec::new();
@@ -957,9 +1024,9 @@ where
 
     let conn = conn.clone();
     // No tick at t=0: a job that finishes within the first interval issues none of the
-    // ping/metric statements below. Cancels are picked up by the first job ping (500 ms in,
-    // 2 s on agent workers). Memory is still sampled at start, without SQL: it is the only
-    // `mem_peak` reading a short job gets, and the completed job row reports it.
+    // ping/metric statements below. Cancels are picked up by the next job ping. Memory is still
+    // sampled at start, without SQL: it is the only `mem_peak` reading a short job gets,
+    // and the completed job row reports it.
     let mut interval = interval_at(Instant::now() + update_job_interval, update_job_interval);
     interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
     *mem_peak = (*mem_peak).max(get_mem.next().await.unwrap_or(0));
@@ -1023,11 +1090,10 @@ where
                             }
                         }
                     }
-                    if matches!(conn, Connection::Http(_)) {
-                        if i % 4 != 0 {
-                            // only ping every 4th time (2s) on http agent mode
-                            continue;
-                        }
+                    // An agent worker pays an HTTP round trip per ping, so it pings every 2 s for
+                    // the first 10 s, then every 5 s.
+                    if matches!(conn, Connection::Http(_)) && i % if i < 20 { 4 } else { 10 } != 0 {
+                        continue;
                     }
                     let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| {
                         tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e);
@@ -1074,8 +1140,8 @@ fn child_joined_output_stream(
         .take()
         .expect("child did not have a handle to stdout");
 
-    let stdout = BufReader::new(stdout).lines();
-    let stderr = BufReader::new(stderr).lines();
+    let stdout = BufReader::new(stdout).split(b'\n');
+    let stderr = BufReader::new(stderr).split(b'\n');
     stream::select(
         lines_to_stream(stderr, true, job_id.clone(), w_id.clone())
             .map(|l| l.map(|line| OutputLine { stderr: true, line })),
@@ -1085,18 +1151,32 @@ fn child_joined_output_stream(
 }
 
 pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
-    mut lines: tokio::io::Lines<R>,
+    mut lines: tokio::io::Split<R>,
     stderr: bool,
     job_id: Uuid,
     w_id: String,
 ) -> impl futures::Stream<Item = io::Result<String>> {
     stream::poll_fn(move |cx| {
         std::pin::Pin::new(&mut lines)
-            .poll_next_line(cx)
-            .map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id))
+            .poll_next_segment(cx)
+            .map(|result| {
+                let result = result.map(|segment| segment.map(decode_output_line));
+                process_streaming_log_lines(result, stderr, &job_id, &w_id)
+            })
     })
 }
 
+/// Child output is not guaranteed to be UTF-8 (e.g. pwsh on Windows writes in the
+/// console code page), so decode lossily: a strict decode errors out and ends log
+/// capture for the rest of the job. Strips a trailing `\r` so CRLF output doesn't keep it.
+pub(crate) fn decode_output_line(mut segment: Vec<u8>) -> String {
+    if segment.last() == Some(&b'\r') {
+        segment.pop();
+    }
+    String::from_utf8(segment)
+        .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())
+}
+
 pub fn process_status(
     program: &str,
     status: ExitStatus,
@@ -1143,4 +1223,33 @@ mod tests {
         let left = JobDeadline(Some(Instant::now() + Duration::from_secs(120)));
         assert!(matches!(left.remaining_secs(), Some(s) if (118..=120).contains(&s)));
     }
+
+    // "Prüfe" in CP850/1252 is not UTF-8. A strict decode errors on that line and
+    // would stop log capture, dropping every line after it.
+    #[tokio::test]
+    async fn a_non_utf8_line_is_decoded_lossily_and_later_lines_still_arrive() {
+        let output: &[u8] =
+            b"start\r\nPr\x81fe Benutzer\nPr\xfcfe\nGr\xc3\xbc\xc3\x9fe\r\nafter\n\nlast";
+        let lines: Vec<String> = lines_to_stream(
+            BufReader::new(output).split(b'\n'),
+            false,
+            Uuid::nil(),
+            "w".into(),
+        )
+        .map(|line| line.expect("reading from a byte slice cannot fail"))
+        .collect()
+        .await;
+        assert_eq!(
+            lines,
+            vec![
+                "start",
+                "Pr\u{FFFD}fe Benutzer",
+                "Pr\u{FFFD}fe",
+                "Grüße",
+                "after",
+                "",
+                "last",
+            ]
+        );
+    }
 }

backend/windmill-worker/src/php_executor.rs

@@ -17,6 +17,7 @@ use windmill_queue::MiniPulledJob;
 use windmill_parser::{MainArgSignature, Typ};
 use windmill_queue::{append_logs, CanceledBy};
 
+use crate::common::DEV_CONF_NSJAIL;
 use crate::{
     common::{
         build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file,
@@ -24,11 +25,13 @@ use crate::{
         resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics,
     },
     handle_child::handle_child,
-    is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH,
+    is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
+    PHP_PATH, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH,
 };
 use windmill_common::client::AuthedClient;
 
 const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto");
+const NSJAIL_CONFIG_LOCK_PHP_CONTENT: &str = include_str!("../nsjail/lock.php.config.proto");
 
 lazy_static::lazy_static! {
     static ref RE: Regex = Regex::new(r"^//\s?(\S+)\s*$").unwrap();
@@ -199,15 +202,99 @@ pub async fn composer_install(
         write_file(job_dir, "composer.lock", lock)?;
     }
 
-    let mut child_cmd = Command::new(&*COMPOSER_PATH);
-    let args = vec!["install", "--no-dev", "--no-progress"];
-    child_cmd
-        .current_dir(job_dir)
-        .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
-        .args(args)
-        .stdout(Stdio::piped())
-        .stderr(Stdio::piped());
-    let child_process = start_child_process(child_cmd, &*COMPOSER_PATH, false).await?;
+    // `composer install` runs the plugin and script code of installed packages, so
+    // when sandboxing is on it is confined under nsjail like the run step. Inside the
+    // jail the environment is cleared down to what composer needs: PATH, COMPOSER_HOME
+    // (pointed at the mounted cache dir), the proxies, the operator-controlled
+    // WHITELIST_ENVS, and every other COMPOSER_* var from the worker env — Windmill has
+    // no managed composer-registry setting, so operators supply private Packagist auth
+    // and composer tuning (COMPOSER_AUTH, COMPOSER_PROCESS_TIMEOUT, …) through those.
+    // COMPOSER_CACHE_DIR is dropped so it can't point composer outside the jail, and a
+    // custom COMPOSER_CAFILE/COMPOSER_CAPATH is bind-mounted so TLS to private feeds
+    // keeps working. COMPOSER_PATH is Windmill-internal. Without sandboxing composer
+    // runs directly, with no isolation.
+    let composer_args = ["install", "--no-dev", "--no-progress"];
+    let child_cmd = if is_sandboxing_enabled() {
+        // The cache dir is bind-mounted into the jail, so it must exist before
+        // nsjail builds the mount tree.
+        std::fs::create_dir_all(&*COMPOSER_CACHE_DIR)?;
+        let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await;
+        // A custom CA bundle/dir lives outside the jail's mounts; bind it in so
+        // composer's TLS can read it (mandatory: false → skipped when the path is unset).
+        let ca_mount = ["COMPOSER_CAFILE", "COMPOSER_CAPATH"]
+            .iter()
+            .filter_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty()))
+            .map(|p| {
+                format!(
+                    "mount {{\n    src: \"{p}\"\n    dst: \"{p}\"\n    is_bind: true\n    mandatory: false\n}}\n"
+                )
+            })
+            .collect::<String>();
+        write_file(
+            job_dir,
+            "lock.config.proto",
+            NSJAIL_CONFIG_LOCK_PHP_CONTENT
+                .replace("{TIMEOUT}", &nsjail_timeout)
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{COMPOSER_CACHE_DIR}", &*COMPOSER_CACHE_DIR)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
+                .replace("{CA_MOUNT}", &ca_mount)
+                .replace("#{DEV}", DEV_CONF_NSJAIL)
+                .replace(
+                    "{TMP_MOUNT_BLOCK}",
+                    &resolve_nsjail_tmp_mount_block(job_dir).await,
+                )
+                .as_str(),
+        )?;
+
+        let composer_passthrough_envs: Vec<(String, String)> = std::env::vars()
+            .filter(|(k, _)| {
+                k.starts_with("COMPOSER_")
+                    && k != "COMPOSER_HOME"
+                    && k != "COMPOSER_CACHE_DIR"
+                    && k != "COMPOSER_PATH"
+            })
+            .collect();
+
+        let mut nsjail_args = vec![
+            "--config",
+            "lock.config.proto",
+            "--",
+            COMPOSER_PATH.as_str(),
+        ];
+        nsjail_args.extend(composer_args);
+
+        let mut child_cmd = Command::new(NSJAIL_PATH.as_str());
+        child_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            // HOME is set to /tmp by the nsjail config; keep_env passes the rest.
+            .env("PATH", PATH_ENV.as_str())
+            .envs(PROXY_ENVS.clone())
+            .envs(&*crate::worker::WHITELIST_ENVS)
+            .envs(composer_passthrough_envs)
+            .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
+            .args(nsjail_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        child_cmd
+    } else {
+        let mut child_cmd = Command::new(&*COMPOSER_PATH);
+        child_cmd
+            .current_dir(job_dir)
+            .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR)
+            .args(composer_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        child_cmd
+    };
+    let child_executable = if is_sandboxing_enabled() {
+        NSJAIL_PATH.as_str()
+    } else {
+        COMPOSER_PATH.as_str()
+    };
+    let child_process = start_child_process(child_cmd, child_executable, false).await?;
 
     handle_child(
         job_id,
@@ -215,7 +302,8 @@ pub async fn composer_install(
         mem_peak,
         canceled_by,
         child_process,
-        false,
+        // jailed when sandboxing is on, so mem sampling finds the real process
+        is_sandboxing_enabled(),
         worker_name,
         w_id,
         "composer install",

backend/windmill-worker/src/python_executor.rs

@@ -116,6 +116,7 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver<Pipt
 
 const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
 const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
+const NSJAIL_CONFIG_LOCK_PY_CONTENT: &str = include_str!("../nsjail/lock.py.config.proto");
 pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
 
 /// Every file exchanged with a job is UTF-8 by construction, so the interpreter
@@ -156,7 +157,7 @@ use crate::{
     },
     get_proxy_envs_for_lang,
     handle_child::handle_child,
-    is_sandboxing_enabled, read_ee_registry_with_workspace_override,
+    is_sandboxing_enabled, read_ee_registry,
     worker_utils::ping_job_status,
     PyV, DISABLE_NUSER, HOME_ENV, INDEX_CERT, NATIVE_CERT, NSJAIL_AVAILABLE, NSJAIL_PATH,
     NSJAIL_PY_RLIMIT_AS_MB, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS,
@@ -279,7 +280,12 @@ pub async fn uv_pip_compile(
     #[cfg(feature = "enterprise")]
     let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?;
 
-    let ws_suffix = crate::workspace_registry_cache_suffix(w_id).await;
+    let pip_indexes = PipIndexes::snapshot(w_id).await;
+    let ws_suffix = if pip_indexes.ws_scoped {
+        format!(":ws:{w_id}")
+    } else {
+        String::new()
+    };
     let exclude_newer_suffix = uv_exclude_newer
         .as_deref()
         .map(|v| format!("-en{}", v))
@@ -343,31 +349,12 @@ pub async fn uv_pip_compile(
         if no_cache {
             args.extend(["--no-cache"]);
         }
-        let pip_extra_index_url = read_ee_registry_with_workspace_override(
-            PIP_EXTRA_INDEX_URL.read().await.clone(),
-            "pip_extra_index_url",
-            "pip extra index url",
-            job_id,
-            w_id,
-            conn,
-        )
-        .await
-        .map(handle_ephemeral_token);
+        let (pip_extra_index_url, pip_index_url) = pip_indexes.urls(job_id, w_id, conn).await;
         if let Some(url) = pip_extra_index_url.as_ref() {
             url.split(",").for_each(|url| {
                 args.extend(["--extra-index-url", url]);
             });
         }
-        let pip_index_url = read_ee_registry_with_workspace_override(
-            PIP_INDEX_URL.read().await.clone(),
-            "pip_index_url",
-            "pip index url",
-            job_id,
-            w_id,
-            conn,
-        )
-        .await
-        .map(handle_ephemeral_token);
         if let Some(url) = pip_index_url.as_ref() {
             args.extend(["--index-url", url]);
         }
@@ -393,18 +380,76 @@ pub async fn uv_pip_compile(
         #[cfg(unix)]
         let uv_cmd = UV_PATH.as_str();
 
-        let mut child_cmd = Command::new(uv_cmd);
-        child_cmd
-            .current_dir(job_dir)
-            .env_clear()
-            .env("HOME", HOME_ENV.to_string())
-            .env("PATH", PATH_ENV.to_string())
-            .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string())
-            .env("UV_INDEX_STRATEGY", uv_index_strategy)
-            .envs(PROXY_ENVS.clone())
-            .args(&args)
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
+        // Resolving an sdist whose metadata is not static makes uv invoke its
+        // PEP 517 build backend, which runs arbitrary package code (e.g.
+        // setup.py). When sandboxing is on, confine that under nsjail exactly
+        // like the install and run steps already are — otherwise it runs as the
+        // worker, in the worker's namespaces, with the worker's env and tokens
+        // readable. The jail gets network to the index, the job dir and the
+        // shared uv cache writable, and the managed interpreter read-only.
+        let mut child_cmd = if is_sandboxing_enabled() {
+            let index_cert_mount = INDEX_CERT
+                .as_ref()
+                .map(|p| {
+                    format!(
+                        "mount {{\n    src: \"{p}\"\n    dst: \"{p}\"\n    is_bind: true\n    mandatory: false\n}}\n"
+                    )
+                })
+                .unwrap_or_default();
+            // The cache dir is bind-mounted into the jail, so it must exist before
+            // nsjail builds the mount tree (uv would otherwise create it itself).
+            std::fs::create_dir_all(&*UV_CACHE_DIR)?;
+            let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await;
+            write_file(
+                job_dir,
+                "lock.config.proto",
+                NSJAIL_CONFIG_LOCK_PY_CONTENT
+                    .replace("{TIMEOUT}", &nsjail_timeout)
+                    .replace("{JOB_DIR}", job_dir)
+                    .replace("{UV_CACHE_DIR}", &*UV_CACHE_DIR)
+                    .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR)
+                    .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                    .replace("{INDEX_CERT_MOUNT}", &index_cert_mount)
+                    .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH)
+                    .replace("#{DEV}", DEV_CONF_NSJAIL)
+                    .replace(
+                        "{TMP_MOUNT_BLOCK}",
+                        &resolve_nsjail_tmp_mount_block(job_dir).await,
+                    )
+                    .as_str(),
+            )?;
+
+            let mut nsjail_args = vec!["--config", "lock.config.proto", "--", uv_cmd];
+            nsjail_args.extend(args.iter().copied());
+
+            let mut child_cmd = Command::new(NSJAIL_PATH.as_str());
+            child_cmd
+                .current_dir(job_dir)
+                .env_clear()
+                // HOME is set to /tmp by the nsjail config; keep_env passes the rest.
+                .env("PATH", PATH_ENV.to_string())
+                .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string())
+                .env("UV_INDEX_STRATEGY", uv_index_strategy)
+                .envs(PROXY_ENVS.clone())
+                .args(nsjail_args)
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped());
+            child_cmd
+        } else {
+            let mut child_cmd = Command::new(uv_cmd);
+            child_cmd
+                .current_dir(job_dir)
+                .env_clear()
+                .env("HOME", HOME_ENV.to_string())
+                .env("PATH", PATH_ENV.to_string())
+                .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string())
+                .env("UV_INDEX_STRATEGY", uv_index_strategy)
+                .envs(PROXY_ENVS.clone())
+                .args(&args)
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped());
+            child_cmd
+        };
 
         if let Some(mirror) = UV_PYTHON_INSTALL_MIRROR.read().await.as_ref() {
             child_cmd.env("UV_PYTHON_INSTALL_MIRROR", mirror);
@@ -454,7 +499,12 @@ pub async fn uv_pip_compile(
                 );
         }
 
-        let child_process = start_child_process(child_cmd, uv_cmd, false).await?;
+        let child_executable = if is_sandboxing_enabled() {
+            NSJAIL_PATH.as_str()
+        } else {
+            uv_cmd
+        };
+        let child_process = start_child_process(child_cmd, child_executable, false).await?;
         append_logs(&job_id, &w_id, logs, conn).await;
         handle_child(
             job_id,
@@ -462,7 +512,8 @@ pub async fn uv_pip_compile(
             mem_peak,
             canceled_by,
             child_process,
-            false,
+            // jailed when sandboxing is on, so mem sampling finds the real process
+            is_sandboxing_enabled(),
             worker_name,
             &w_id,
             // TODO: Rename to uv-pip-compile?
@@ -2068,6 +2119,62 @@ Returned from server: py_version - {:?}, py_version_v2 - {:?}
     Ok((pyv, additional_python_paths))
 }
 
+/// The pip index settings a job resolves and installs from, and whether its workspace
+/// overrides any registry.
+///
+/// Both come from one snapshot of the workspace's overrides: read separately, a settings
+/// reload in between could file an install from an overridden index under a shared cache name.
+struct PipIndexes {
+    extra_index_url: Option<String>,
+    index_url: Option<String>,
+    /// Caches filled through these URLs must not be shared with other workspaces.
+    ws_scoped: bool,
+}
+
+impl PipIndexes {
+    async fn snapshot(w_id: &str) -> Self {
+        let overrides = crate::workspace_registry_overrides(w_id).await;
+        let ws_value = |key: &str| match overrides.get(key) {
+            Some(serde_json::Value::String(s)) => Some(s.clone()),
+            _ => None,
+        };
+        // An empty value means unset, and an empty workspace override unsets the global one.
+        Self {
+            extra_index_url: ws_value("pip_extra_index_url")
+                .or(PIP_EXTRA_INDEX_URL.read().await.clone())
+                .filter(|s| !s.trim().is_empty()),
+            index_url: ws_value("pip_index_url")
+                .or(PIP_INDEX_URL.read().await.clone())
+                .filter(|s| !s.trim().is_empty()),
+            ws_scoped: !overrides.is_empty(),
+        }
+    }
+
+    /// The `(extra index, index)` URLs to hand uv. Kept apart from [`Self::snapshot`]: it runs
+    /// `EPHEMERAL_TOKEN_CMD`, which a caller served from a cache has no reason to pay for.
+    async fn urls(
+        self,
+        job_id: &Uuid,
+        w_id: &str,
+        conn: &Connection,
+    ) -> (Option<String>, Option<String>) {
+        (
+            read_ee_registry(
+                self.extra_index_url,
+                "pip extra index url",
+                job_id,
+                w_id,
+                conn,
+            )
+            .await
+            .map(handle_ephemeral_token),
+            read_ee_registry(self.index_url, "pip index url", job_id, w_id, conn)
+                .await
+                .map(handle_ephemeral_token),
+        )
+    }
+}
+
 lazy_static::lazy_static! {
     static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap();
 
@@ -2527,28 +2634,18 @@ pub async fn handle_python_reqs(
         job_id
     );
 
-    let pip_indexes = (
-        read_ee_registry_with_workspace_override(
-            PIP_EXTRA_INDEX_URL.read().await.clone(),
-            "pip_extra_index_url",
-            "pip extra index url",
-            job_id,
-            w_id,
-            conn,
-        )
-        .await
-        .map(handle_ephemeral_token),
-        read_ee_registry_with_workspace_override(
-            PIP_INDEX_URL.read().await.clone(),
-            "pip_index_url",
-            "pip index url",
-            job_id,
-            w_id,
-            conn,
-        )
-        .await
-        .map(handle_ephemeral_token),
-    );
+    let pip_indexes = PipIndexes::snapshot(w_id).await;
+    let ws_scoped = pip_indexes.ws_scoped;
+    let pip_indexes = pip_indexes.urls(job_id, w_id, conn).await;
+    // The wheel dir is shared by every workspace on the worker, and by every worker through
+    // the object store, under a name that says nothing of the index it was filled from. A
+    // workspace with its own registries gets its own, as its resolution cache does. The id
+    // is hashed: it is not restricted to characters safe in a file name.
+    let ws_dir_suffix = if ws_scoped {
+        format!("-ws-{}", &calculate_hash(w_id)[..16])
+    } else {
+        String::new()
+    };
 
     // Cached paths
     let mut req_with_penv: Vec<(String, String)> = vec![];
@@ -2577,7 +2674,7 @@ pub async fn handle_python_reqs(
         let py_prefix = &py_version.to_cache_dir(false);
 
         let venv_p = format!(
-            "{py_prefix}/{}",
+            "{py_prefix}/{}{ws_dir_suffix}",
             req.replace(' ', "").replace('/', "").replace(':', "")
         );
         if metadata(venv_p.clone() + "/.valid.windmill").await.is_ok() {

backend/windmill-worker/src/snowflake_executor.rs

@@ -855,7 +855,7 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value {
                 }
             })
         }
-        "binary" => {
+        "binary" | "varbinary" => {
             // convert base64 to hex as expected by snowflake
             let bytes = engine::general_purpose::STANDARD
                 .decode(arg_v.as_str().unwrap_or(""))

backend/windmill-worker/src/worker.rs

@@ -155,7 +155,7 @@ use crate::{
         handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
         tally_unfinished_dependency_deploy,
     },
-    worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full},
+    worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full},
 };
 
 #[cfg(feature = "rust")]
@@ -949,6 +949,22 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String {
     }
 }
 
+/// The registry overrides of a workspace, empty when it has none.
+///
+/// One snapshot: registry settings reload under running jobs, so a caller whose cache key
+/// has to agree with the registry values it used reads both from here rather than calling
+/// [`workspace_registry_cache_suffix`] and a `read_ee_registry_*` helper separately.
+pub async fn workspace_registry_overrides(
+    w_id: &str,
+) -> std::collections::HashMap<String, serde_json::Value> {
+    let registries = WORKSPACE_REGISTRIES.read().await;
+    registries
+        .as_ref()
+        .and_then(|m| m.get(w_id))
+        .cloned()
+        .unwrap_or_default()
+}
+
 /// The name a build artifact is cached under, derived from `base` — the runnable's own
 /// cache-key input — and its inline modules.
 ///
@@ -2005,6 +2021,7 @@ pub fn create_span_with_name(
         root_job = field::Empty,
         workspace_id = %arc_job.workspace_id,
         worker = %worker_name,
+        worker_group = %*WORKER_GROUP,
         hostname = field::Empty,
         tag = %arc_job.tag,
         language = field::Empty,
@@ -2156,6 +2173,7 @@ pub fn log_context_for_job(
         job_id: Some(arc_job.id.to_string()),
         workspace_id: Some(arc_job.workspace_id.clone()),
         worker: Some(worker_name.to_string()),
+        worker_group: Some(WORKER_GROUP.clone()),
         tag: Some(arc_job.tag.clone()),
         job_kind: Some(arc_job.kind.as_str().to_string()),
         created_by: Some(arc_job.created_by.clone()),
@@ -2695,7 +2713,7 @@ pub async fn run_worker(
     let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
 
     let mut reported_ip = cached_ip();
-    let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn)
+    let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn)
         .await
         .expect("initial ping could be sent");
 
@@ -3142,6 +3160,31 @@ pub async fn run_worker(
 
     let mut killpill_rx2 = killpill_rx.resubscribe();
 
+    // The loop below only sees the killpill once the running job is done, which is too late
+    // for that job to learn its worker is going away.
+    {
+        let mut killpill_rx = killpill_rx.resubscribe();
+        let conn = conn.clone();
+        let worker_name = worker_name.clone();
+        tokio::spawn(async move {
+            if matches!(
+                killpill_rx.recv().await,
+                Err(broadcast::error::RecvError::Closed)
+            ) {
+                return;
+            }
+            for attempt in 1..=3 {
+                match mark_worker_draining(&conn, &worker_name).await {
+                    Ok(()) => return,
+                    Err(e) => {
+                        tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}");
+                        tokio::time::sleep(Duration::from_secs(2)).await;
+                    }
+                }
+            }
+        });
+    }
+
     loop {
         let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst);
         if last_processing_duration_secs > 5 {
@@ -3217,7 +3260,7 @@ pub async fn run_worker(
             // which costs at most the last job's id here: no job of this worker is in flight at this
             // point in the loop, and the next one refills them.
             if ip_just_resolved && conn.as_sql().is_none() {
-                if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await {
+                if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await {
                     tracing::warn!(
                         worker = %worker_name, hostname = %hostname,
                         "failed to re-register with the resolved external IP: {e}"
@@ -3406,6 +3449,10 @@ pub async fn run_worker(
                     tokio::time::sleep(Duration::from_millis(200)).await;
                     continue;
                 }
+            } else if WORKER_CONFIG.load().paused {
+                // Checked after the same-worker channel: a flow this worker already runs
+                // still needs it for its remaining same-worker steps.
+                Ok(None)
             } else {
                 match &conn {
                     Connection::Sql(db) => {

backend/windmill-worker/src/worker_flow.rs

@@ -192,6 +192,7 @@ pub async fn update_flow_status_after_job_completion(
         flow_job_duration,
         stop_early_override,
         has_triggered_error_handler: false,
+        recovered_by_child: false,
     };
     let mut step_failure = step_failure;
     loop {
@@ -211,6 +212,7 @@ pub async fn update_flow_status_after_job_completion(
             worker_dir,
             rec.stop_early_override,
             rec.has_triggered_error_handler,
+            rec.recovered_by_child,
             worker_name,
             job_completed_tx.clone(),
             flow_runners.clone(),
@@ -240,6 +242,7 @@ pub async fn update_flow_status_after_job_completion(
                     worker_dir,
                     rec.stop_early_override,
                     rec.has_triggered_error_handler,
+                    false,
                     worker_name,
                     job_completed_tx.clone(),
                     flow_runners.clone(),
@@ -293,6 +296,10 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
     flow_job_duration: Option<FlowJobDuration>,
     stop_early_override: Option<bool>,
     has_triggered_error_handler: bool,
+    /// The failed child's error handler returned `recover: true`. The parent still handles the
+    /// failure as it would any other, so the same steps run; only a root flow that this failure
+    /// ends turns green.
+    recovered_by_child: bool,
 }
 
 #[derive(Deserialize)]
@@ -427,6 +434,7 @@ pub async fn update_flow_status_after_job_completion_internal(
     worker_dir: &str,
     stop_early_override: Option<bool>,
     has_triggered_error_handler: bool,
+    mut recovered_by_child: bool,
     worker_name: &str,
     job_completed_tx: JobCompletedSender,
     flow_runners: Option<Arc<FlowRunners>>,
@@ -449,6 +457,7 @@ pub async fn update_flow_status_after_job_completion_internal(
         skip_if_stop_early,
         nresult,
         is_failure_step,
+        failure_step_recovers,
         _cleanup_module,
         chat_ai_info,
     ) = {
@@ -1096,6 +1105,9 @@ pub async fn update_flow_status_after_job_completion_internal(
                     } else {
                         has_triggered_error_handler = false;
                     }
+                    // Recovered iterations are stored as successes, so a failed parallel
+                    // node failed on an iteration that did not recover.
+                    recovered_by_child = false;
                     (success, Some(new_status))
                 } else {
                     add_time!(bench, "handle parallel flow start");
@@ -1823,6 +1835,13 @@ pub async fn update_flow_status_after_job_completion_internal(
             _ => false,
         };
 
+        let failure_step_recovers = is_failure_step && result_has_recover_true(nresult.clone());
+        // An error this flow raises on its own (a stop predicate's error message) is a new
+        // failure that no error handler recovered.
+        if stop_early_err_msg.is_some() {
+            recovered_by_child = false;
+        }
+
         let chat_ai_info = ChatAiInfo {
             chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false),
             conversation_id: old_status.memory_id,
@@ -1837,6 +1856,7 @@ pub async fn update_flow_status_after_job_completion_internal(
             skip_if_stop_early,
             nresult,
             is_failure_step,
+            failure_step_recovers,
             old_status.cleanup_module,
             chat_ai_info,
         )
@@ -1844,6 +1864,10 @@ pub async fn update_flow_status_after_job_completion_internal(
 
     let flow_job = Arc::new(flow_job);
 
+    // An inner flow that ends on a recovered failure stays failed, as parents read their
+    // children's stored status (a parallel node counts its iterations' statuses), so only the
+    // root flow turns green.
+    let ends_root = !(flow_job.is_flow_step() && flow_job.parent_job.is_some());
     let done = if !should_continue_flow {
         {
             let logs = if flow_job.is_canceled() {
@@ -1851,7 +1875,9 @@ pub async fn update_flow_status_after_job_completion_internal(
             } else if stop_early {
                 format!("Flow job stopped early because of a stop early predicate returning true\n")
             } else if is_failure_step {
-                format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, result_has_recover_true(nresult.clone()))
+                format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, failure_step_recovers)
+            } else if !success && recovered_by_child && ends_root {
+                format!("Flow job completed with success: the step that failed had its error recovered by the error handler (recover: true)\n")
             } else {
                 format!(
                     "Flow job completed with {}\n",
@@ -2048,7 +2074,11 @@ pub async fn update_flow_status_after_job_completion_internal(
                 .await;
             }
 
-            let success = success && (!is_failure_step || result_has_recover_true(nresult.clone()));
+            let success = if is_failure_step {
+                success && failure_step_recovers
+            } else {
+                success || (recovered_by_child && ends_root)
+            };
 
             add_time!(bench, "flow status update 1");
 
@@ -2149,6 +2179,8 @@ pub async fn update_flow_status_after_job_completion_internal(
                 .await;
                 // override prior child's success so the parent learns this flow failed
                 success = false;
+                // a new failure, which no error handler recovered
+                recovered_by_child = false;
                 true
             }
             Ok(_) => false,
@@ -2178,6 +2210,7 @@ pub async fn update_flow_status_after_job_completion_internal(
                             None
                         },
                         has_triggered_error_handler: has_triggered_error_handler || is_failure_step,
+                        recovered_by_child: failure_step_recovers || (recovered_by_child && !success),
                     },
                 ));
             }

backend/windmill-worker/src/worker_utils.rs

@@ -7,9 +7,9 @@ use windmill_common::{
     external_ip::UNKNOWN_IP,
     worker::{
         get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
-        insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
-        update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
-        WORKER_CONFIG, WORKER_GROUP,
+        insert_ping_query, set_worker_draining_query, update_job_ping_query,
+        update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping,
+        PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP,
     },
     KillpillSender, DB,
 };
@@ -164,6 +164,7 @@ async fn update_worker_ping_full_inner(
                         wm_memory_usage: get_windmill_memory_usage(),
                         job_isolation: None,
                         native_mode: Some(native_mode),
+                        draining: None,
                         ping_type: PingType::MainLoop,
                     },
                 )
@@ -179,6 +180,7 @@ pub async fn insert_ping(
     worker_instance: &str,
     worker_name: &str,
     ip: Option<&str>,
+    reset_draining: bool,
     db: &Connection,
 ) -> anyhow::Result<i32> {
     let (tags, dw, dws, native_mode) = {
@@ -224,6 +226,7 @@ pub async fn insert_ping(
                 memory,
                 job_isolation,
                 native_mode,
+                reset_draining.then_some(false),
                 db,
             )
             .await;
@@ -256,6 +259,7 @@ pub async fn insert_ping(
                         wm_memory_usage: get_windmill_memory_usage(),
                         job_isolation,
                         native_mode: Some(native_mode),
+                        draining: reset_draining.then_some(false),
                         ping_type: PingType::Initial,
                     },
                 )
@@ -314,6 +318,7 @@ pub async fn update_worker_ping_from_job(
                     &Ping {
                         last_job_executed: Some(job_id.clone()),
                         last_job_workspace_id: Some(w_id.to_string()),
+                        draining: None,
                         ping_type: PingType::Job,
                         worker_instance: None,
                         ip: None,
@@ -342,6 +347,45 @@ pub async fn update_worker_ping_from_job(
     Ok(())
 }
 
+/// Publishes on the worker's ping row that it has received its killpill, so the job it is
+/// still running can find out and exit early.
+pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> {
+    match conn {
+        Connection::Sql(db) => set_worker_draining_query(worker_name, db).await,
+        Connection::Http(client) => {
+            client
+                .post::<Ping, ()>(
+                    UPDATE_PING_URL,
+                    None,
+                    &Ping {
+                        last_job_executed: None,
+                        last_job_workspace_id: None,
+                        draining: None,
+                        ping_type: PingType::Draining,
+                        worker_instance: None,
+                        ip: None,
+                        tags: None,
+                        dw: None,
+                        dws: None,
+                        version: None,
+                        vcpus: None,
+                        memory: None,
+                        memory_usage: None,
+                        wm_memory_usage: None,
+                        jobs_executed: None,
+                        occupancy_rate: None,
+                        occupancy_rate_15s: None,
+                        occupancy_rate_5m: None,
+                        occupancy_rate_30m: None,
+                        job_isolation: None,
+                        native_mode: None,
+                    },
+                )
+                .await
+        }
+    }
+}
+
 pub async fn ping_job_status(
     conn: &Connection,
     job_id: &Uuid,