starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-13 08:02:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into migrate-session-to-fork

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
This commit is contained in:
Guilhem Lemouel
2026-10-07 07:42:03 -07:00
parent b52b0ca3eb f2fb15603b
commit ca78daca7d
259 changed files with 9724 additions and 2131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+43 -29
View File
@@ -1,25 +1,22 @@
name: AI Evals (scheduled)
# Full ai_evals suites on current models, posted to the AI evals dashboard in
# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
# tokens per model: every suite runs on MODELS, and global (the mode users get)
# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by
# hand to measure a branch against main.
# The global (AI chat) and cli suites on current models, posted to the AI evals
# dashboard in the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is
# tracked over time. Weekly: one global pass at --runs 3 is about 45M tokens per
# model. Each mode and model is its own job, so a slow model neither delays nor
# shares a timeout with the others. Run it by hand to measure a branch against
# main, or to include the flow, app and script suites.
on:
schedule:
- cron: "0 3 * * 1"
workflow_dispatch:
inputs:
modes:
description: "Space-separated modes"
default: "global flow app script cli"
description: "Space-separated modes (global, cli, flow, app, script)"
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models)"
default: "sonnet-5.5"
global_extra_models:
description: "Extra model aliases for global mode only"
default: "gpt-6-astra gemini-3.8-flash"
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
default: "3"
@@ -31,8 +28,7 @@ concurrency:
group: ai-evals-scheduled-${{ github.ref }}
env:
MODELS: ${{ inputs.models || 'sonnet-5.5' }}
GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }}
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }}
RUNS: ${{ inputs.runs || '3' }}
REASONING: ${{ inputs.reasoning }}
# Routes on the cloud instance are prefixed with their workspace.
@@ -42,14 +38,31 @@ jobs:
setup:
runs-on: ubuntu-latest
outputs:
modes: ${{ steps.modes.outputs.modes }}
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- id: modes
- uses: actions/checkout@v4
with:
sparse-checkout: ai_evals/core
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.4.0
- id: matrix
env:
MODES: ${{ inputs.modes || 'global flow app script cli' }}
MODES: ${{ inputs.modes || 'global cli' }}
# cli takes only the aliases with a `cli` entry in ai_evals/core/models.ts.
run: |
echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ")
| map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT"
cli_models=$(bun -e '
import { EVAL_MODELS } from "./ai_evals/core/models.ts";
console.log(JSON.stringify(EVAL_MODELS.filter((m) => m.cli)
.flatMap((m) => [m.id, ...m.aliases]).map((a) => a.toLowerCase())));')
echo "matrix=$(jq -cn --arg modes "$MODES" --arg models "$MODELS" --argjson cli "$cli_models" '
def words: split(" ") | map(select(. != ""));
[ $modes | words[] | select(IN("global", "flow", "app", "script", "cli")) as $mode
| $models | words[]
| select($mode != "cli" or (ascii_downcase | IN($cli[])))
| {mode: $mode, model: .} ]')" >> "$GITHUB_OUTPUT"
evals:
needs: setup
@@ -58,7 +71,7 @@ jobs:
strategy:
fail-fast: false
matrix:
mode: ${{ fromJSON(needs.setup.outputs.modes) }}
include: ${{ fromJSON(needs.setup.outputs.matrix) }}
services:
postgres:
image: postgres:16
@@ -118,10 +131,15 @@ jobs:
working-directory: ./cli
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
- name: Run ${{ matrix.mode }} evals
- name: Install Codex CLI
if: matrix.mode == 'cli'
run: npm install --global @openai/codex@0.159.3
- name: Run ${{ matrix.mode }} evals on ${{ matrix.model }}
working-directory: ./ai_evals
env:
MODE: ${{ matrix.mode }}
MODEL: ${{ matrix.model }}
WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -131,14 +149,10 @@ jobs:
run: |
bun install
mkdir -p results
models="$MODELS"
[ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS"
reasoning=()
[ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING")
for m in $models; do
bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \
--output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored"
done
bun run cli -- run "$MODE" --model "$MODEL" --runs "$RUNS" "${reasoning[@]}" \
--output "$PWD/results/$MODE-$MODEL.json" || echo "::warning::$MODE on $MODEL errored"
- name: Post results to the dashboard
if: always()
@@ -166,7 +180,7 @@ jobs:
uses: actions/upload-artifact@v4
if: always()
with:
name: ai-evals-${{ matrix.mode }}
name: ai-evals-${{ matrix.mode }}-${{ matrix.model }}
path: |
ai_evals/logs
ai_evals/results
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.823.1"
".": "1.826.0"
}
AGENTS.md
+6
View File
@@ -90,6 +90,12 @@ are for a plain checkout with nothing running.
- **Login**: `admin@windmill.dev` / `changeme`
- **Instance settings**: navigate to `/#superadmin-settings`
- **Migrations**: use `cargo sqlx migrate add -r <name>` from `backend/` to create new migrations (never generate timestamps manually)
- **Resource types ship through the hub, not migrations.** A new type, a provider's credential
included, is a `hub/<app>/<app>.resource_type.json` in `windmill-labs/windmill-integrations`
and reaches instances by hub sync; a product name its identifier misses (`typesafe` →
"TypeSafe") is that file's `display_name`, not a word in `resourceTypeDisplay.ts`'s map. Only a
type Windmill itself defines and reads as a product object (`ai_skill`, `ai_instruction`) is
also seeded by a migration, so the feature works before a sync. For local testing, insert the row.
### Per-worktree ports and database
CHANGELOG.md
+72
View File
@@ -1,5 +1,77 @@
# Changelog
## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07)
### Features
* count OIDC signatures and reuse server-side tokens ([#11556](https://github.com/windmill-labs/windmill/issues/11556)) ([e277735](https://github.com/windmill-labs/windmill/commit/e277735788c260239a063063459776ec01f7d907))
* isolate the raw-app editor preview when the app opts into sandboxing ([#11572](https://github.com/windmill-labs/windmill/issues/11572)) ([fa25397](https://github.com/windmill-labs/windmill/commit/fa253976e213c1053d1832c1d1fb7bedf7ce8c45))
* job_token_scopes OIDC claim ([#11551](https://github.com/windmill-labs/windmill/issues/11551)) ([d493eff](https://github.com/windmill-labs/windmill/commit/d493eff237f96a353fb6f5715a15bcad61d24433))
* set up a new app in one modal over its AI session, fix its deploy ([#11582](https://github.com/windmill-labs/windmill/issues/11582)) ([7190185](https://github.com/windmill-labs/windmill/commit/71901852fb3cc40515d6675a0a88cd158e942d56))
### Bug Fixes
* export getWorkspace and other client helpers as named exports ([#11581](https://github.com/windmill-labs/windmill/issues/11581)) ([bf201be](https://github.com/windmill-labs/windmill/commit/bf201be11e95f0fc3e12353531f25bb3ceb0f8f3))
* give the dbt workspace settings page a header like its siblings ([#11576](https://github.com/windmill-labs/windmill/issues/11576)) ([38aa4ea](https://github.com/windmill-labs/windmill/commit/38aa4eaa1200db52e67a18ef9a7c5dc4d60a4948))
* ignore schedule missed-run counters in workspace compare ([#11570](https://github.com/windmill-labs/windmill/issues/11570)) ([eab3dd5](https://github.com/windmill-labs/windmill/commit/eab3dd5a52bc8faf57bb84c85bd74eb20ebd8ec7))
* keep chat tool cards collapsed the way the user left them ([#11575](https://github.com/windmill-labs/windmill/issues/11575)) ([c71ecad](https://github.com/windmill-labs/windmill/commit/c71ecad6f9cf3727a305b5f9038dcbe6a54398b8))
* keep run status buttons from selecting the flow input node ([#11579](https://github.com/windmill-labs/windmill/issues/11579)) ([8661797](https://github.com/windmill-labs/windmill/commit/866179730a9798346d3950f27dd625c3358e690a))
* make every trigger and schedule editor control respect read-only mode ([#11569](https://github.com/windmill-labs/windmill/issues/11569)) ([52a99c3](https://github.com/windmill-labs/windmill/commit/52a99c3047e1add477e36919b15a745f7d2604a0))
* strip NUL from ai agent output and job args before jsonb writes ([#11568](https://github.com/windmill-labs/windmill/issues/11568)) ([cf4bec6](https://github.com/windmill-labs/windmill/commit/cf4bec611f6df8a45b238afe5b8da3e3a5748bf9))
## [1.825.0](https://github.com/windmill-labs/windmill/compare/v1.824.1...v1.825.0) (2026-10-06)
### Features
* choose build with AI or code when creating a raw app ([#11560](https://github.com/windmill-labs/windmill/issues/11560)) ([431f861](https://github.com/windmill-labs/windmill/commit/431f86138815fce432e282e8ac495953c5ecf98a))
* confine jobs:read to the runnable paths it names ([#11550](https://github.com/windmill-labs/windmill/issues/11550)) ([b371af3](https://github.com/windmill-labs/windmill/commit/b371af31c96e6890fe9223233d2c6e53ddea67d4))
* open a run as a pushed page on the narrow runs page ([#11541](https://github.com/windmill-labs/windmill/issues/11541)) ([fd8de45](https://github.com/windmill-labs/windmill/commit/fd8de45cc7cfcba2c374dd86f87f4d3e8aec6ded))
* say a canceled team plan is canceled, not past due ([#11539](https://github.com/windmill-labs/windmill/issues/11539)) ([18e44d3](https://github.com/windmill-labs/windmill/commit/18e44d31742b8d16852b8c6ae663979666e064d7))
### Bug Fixes
* group raw app edits that carry a file diff in the ai chat ([#11558](https://github.com/windmill-labs/windmill/issues/11558)) ([5912d30](https://github.com/windmill-labs/windmill/commit/5912d300a550a33674fd73740d613d1b434fbba1))
* honor custom timeout in setup phases and name the limit hit ([#11557](https://github.com/windmill-labs/windmill/issues/11557)) ([4fd4f76](https://github.com/windmill-labs/windmill/commit/4fd4f76d0df740cb3eebed1a0625b970484807fc))
* search slow runs filters by time windows instead of timing out ([#11549](https://github.com/windmill-labs/windmill/issues/11549)) ([2bf9cfc](https://github.com/windmill-labs/windmill/commit/2bf9cfcd59e1101695160524c31a4da1a41699dc))
* serve the raw-app preview shell sandboxed, load a blob copy in the editor ([#11554](https://github.com/windmill-labs/windmill/issues/11554)) ([5f92daa](https://github.com/windmill-labs/windmill/commit/5f92daab6d0ecac30dd384e1b0f302af13fdd9ab))
* simplify the run page AI card to a prompt and a chat button ([#11561](https://github.com/windmill-labs/windmill/issues/11561)) ([8c74f75](https://github.com/windmill-labs/windmill/commit/8c74f75de3fa62b07186ca4f56dae4a7e4758034))
## [1.824.1](https://github.com/windmill-labs/windmill/compare/v1.824.0...v1.824.1) (2026-10-05)
### Bug Fixes
* keep monaco's diff engine out of the app shell bundle ([#11542](https://github.com/windmill-labs/windmill/issues/11542)) ([0908dee](https://github.com/windmill-labs/windmill/commit/0908dee897e0f87b97418a23ede3662d96270340))
## [1.824.0](https://github.com/windmill-labs/windmill/compare/v1.823.1...v1.824.0) (2026-10-05)
### Features
* add an AI decision flow step (TypeSafe's Jev, Cloudflare's Clef) ([#11474](https://github.com/windmill-labs/windmill/issues/11474)) ([3cba8a7](https://github.com/windmill-labs/windmill/commit/3cba8a7e4cca3ef20ba73a7dad79d892bdc9fed8))
* add notion oauth provider ([#11528](https://github.com/windmill-labs/windmill/issues/11528)) ([8b2dc1e](https://github.com/windmill-labs/windmill/commit/8b2dc1e46c7f4e3859263f18b686ce6f3ce6fd30))
* **ai-chat:** show per-file diffs for raw app edits ([#11521](https://github.com/windmill-labs/windmill/issues/11521)) ([ec0883a](https://github.com/windmill-labs/windmill/commit/ec0883a627b06b6ba2a393d5cf66ddf6ebc4cd0d))
* bill team-plan service accounts as half a seat ([#11298](https://github.com/windmill-labs/windmill/issues/11298)) ([3aee181](https://github.com/windmill-labs/windmill/commit/3aee1815746f0116f5662f162ad1e234717af90d))
* card layout and load-more for the audit logs page ([#11507](https://github.com/windmill-labs/windmill/issues/11507)) ([af1a29a](https://github.com/windmill-labs/windmill/commit/af1a29a3cb33386f61a759ec0dd5eec0ccefefc2))
* offer audit-log and drive-metadata scopes for google oauth connects ([#11527](https://github.com/windmill-labs/windmill/issues/11527)) ([c4e7c71](https://github.com/windmill-labs/windmill/commit/c4e7c714e455f958d9a927c4bc4eb23256324780))
* offer slack read scopes for files, user groups, emoji and dms ([#11526](https://github.com/windmill-labs/windmill/issues/11526)) ([b5fda7d](https://github.com/windmill-labs/windmill/commit/b5fda7d5af7105fde69beb68410984350af2ac7e))
### Bug Fixes
* bound s3 ai attachment downloads before encoding ([#11534](https://github.com/windmill-labs/windmill/issues/11534)) ([92fbfc0](https://github.com/windmill-labs/windmill/commit/92fbfc03d219544c16e8f7db47a547f9ff402c27))
* collapse queued and running chat tool calls into a waiting row ([#11523](https://github.com/windmill-labs/windmill/issues/11523)) ([4f30982](https://github.com/windmill-labs/windmill/commit/4f30982ce3f7abc64a31890f8e88c31c1c2e0803))
* disable postgres triggers whose replication slot was invalidated ([#11536](https://github.com/windmill-labs/windmill/issues/11536)) ([85248ee](https://github.com/windmill-labs/windmill/commit/85248eee786b633def40c42b6007fc7b907fb09e))
* evaluate field conditions and step test inputs in QuickJS ([#11487](https://github.com/windmill-labs/windmill/issues/11487)) ([c962d37](https://github.com/windmill-labs/windmill/commit/c962d3766b1a250ba960e413556890d5919ebc94))
* flush proxied responses before the dev supervisor closes the client ([#11530](https://github.com/windmill-labs/windmill/issues/11530)) ([cefb195](https://github.com/windmill-labs/windmill/commit/cefb1953764e714d79556e2f8065cfbec78acb86))
* label the default slack scopes and list them first in the connect dialog ([#11529](https://github.com/windmill-labs/windmill/issues/11529)) ([e76781c](https://github.com/windmill-labs/windmill/commit/e76781cc2709fda08403e14c1a0a01957e430db1))
* run each ai eval model in its own job and let the docs tools reach the backend ([#11533](https://github.com/windmill-labs/windmill/issues/11533)) ([222dad1](https://github.com/windmill-labs/windmill/commit/222dad1d8cc9cbb6925fdff709e00a49bb63bc4a))
* show the flow row edit button to builder operators ([#11535](https://github.com/windmill-labs/windmill/issues/11535)) ([6c4e34c](https://github.com/windmill-labs/windmill/commit/6c4e34cd0dd9b4329084214c40695f5915947cc2))
## [1.823.1](https://github.com/windmill-labs/windmill/compare/v1.823.0...v1.823.1) (2026-10-05)
Dockerfile
+2 -2
View File
@@ -142,10 +142,10 @@ FROM ${DEBIAN_IMAGE}
ARG TARGETPLATFORM
ARG POWERSHELL_VERSION=7.5.0
ARG KUBECTL_VERSION=1.36.2
ARG KUBECTL_VERSION=1.37.1
ARG HELM_VERSION=3.21.4
# NOTE: If changing, also change go version in workspace dependencies template at WorkspaceDependenciesEditor.svelte
ARG GO_VERSION=1.26.8
ARG GO_VERSION=1.27.1
ARG APP=/usr/src/app
ARG WITH_POWERSHELL=true
ARG WITH_KUBECTL=true
ai_evals/README.md
+21 -1
View File
@@ -89,7 +89,21 @@ Notes:
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
- `cli` mode runs a coding agent over the workspace, chosen by the alias's `cli` entry; aliases without one are rejected there:
| Agent | Models | Label | Needs |
| --- | --- | --- | --- |
| Claude Code (`@anthropic-ai/claude-agent-sdk`) | Claude aliases | `anthropic:<model>` | `ANTHROPIC_API_KEY` |
| Claude Code on DeepSeek's Anthropic-compatible endpoint | `deepseek-v4-pro` | `claude-code:<model>` | `DEEPSEEK_API_KEY` |
| Codex (`codex exec --json`) | `gpt-6.1-sol`, `gpt-6-astra` | `codex:<model>` | `codex` on `PATH` (CI pins 0.159.3), `OPENAI_API_KEY` |
Codex runs with a throwaway `HOME`, so your own `~/.codex` config and skills stay out of the run. It has no
Skill tool: it finds `.agents/skills` on its own and loads a skill by reading its `SKILL.md`, so the trace
records any command that touches `.agents/skills/<name>/SKILL.md` as a `Skill` call to `<name>`, just
before that command, or just after it when the command mutates something before reading the skill. Its shell commands are recorded as `Bash` and its patches as `Edit`, so the
`invokes skill … before first mutation` checks compare the same things for both agents. Codex has no
turn cap; a run stops at `maxTurns` rounds of tool calls (parallel calls count once) and fails, as a
Claude Code run does at its cap.
- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs
## Case Format
@@ -164,6 +178,12 @@ the decrypted value, exactly as against a real backend. The chat's read path pas
Seed a recognizable secret (the existing fixture uses `sk_live_do_not_leak_me`) and
assert it via `valueExcludes` to catch a leak.
`runtime.userAnswers` scripts the user's side of `askUserQuestion` in a global case: one
answer per question, in the order they are asked, and a question past the last answer is
dismissed, so `userAnswers: []` dismisses every question. Without it the tool reports that
it cannot ask, and the model usually carries on with its own guess, which makes "asks
before creating anything" impossible to assert.
`toolExpect.toolCallArgs` entries support `sharedByAtLeast: <n>`: at least `n` recorded
calls to that tool must carry the same non-blank string in the field. Use it for calls that
have to share an identifier, like two test runs of one chat conversation.
ai_evals/adapters/cli/codex.test.ts
+71
View File
@@ -0,0 +1,71 @@
import { describe, expect, it } from "bun:test";
import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { runCodex, toToolInvocations, unwrapShellCommand } from "./codex";
describe("unwrapShellCommand", () => {
it("strips the login-shell wrapper Codex reports", () => {
expect(unwrapShellCommand(`/usr/bin/zsh -lc 'cat .agents/skills/x/SKILL.md'`)).toBe(
"cat .agents/skills/x/SKILL.md"
);
expect(unwrapShellCommand(`/bin/bash -lc "printf 'hi' > \\"a b\\""`)).toBe(
`printf 'hi' > "a b"`
);
expect(unwrapShellCommand(`/bin/bash -lc 'echo '\\''x'\\'''`)).toBe(`echo 'x'`);
expect(unwrapShellCommand(`bash -lc 'wmill sync push'`)).toBe("wmill sync push");
expect(unwrapShellCommand(`/bin/bash -lc 'rg -g '"'"'!x'"'"' f'`)).toBe(`rg -g '!x' f`);
});
});
describe("toToolInvocations", () => {
it("records reading a SKILL.md as a Skill call before the command", () => {
const tools = toToolInvocations({
type: "command_execution",
command: `/bin/bash -lc 'sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f'`,
});
expect(tools.map((tool) => [tool.tool, tool.input])).toEqual([
["Skill", { skill: "write-script-bun" }],
["Bash", { command: "sed -n 1,200p .agents/skills/write-script-bun/SKILL.md && mkdir f" }],
]);
});
it("records a skill read after a mutation earlier in the same command", () => {
const tools = toToolInvocations({
type: "command_execution",
command: `/bin/bash -lc 'printf main > f/a.ts && cat .agents/skills/write-script-bun/SKILL.md'`,
});
expect(tools.map((tool) => tool.tool)).toEqual(["Bash", "Skill"]);
});
it("maps a patch to Edit so it counts as a mutation", () => {
expect(
toToolInvocations({ type: "file_change", changes: [{ path: "a.ts", kind: "add" }] })[0]
).toMatchObject({ tool: "Edit", input: { changes: [{ path: "a.ts", kind: "add" }] } });
});
});
describe("runCodex", () => {
it("keeps parallel tool calls in the order they started", async () => {
const dir = await mkdtemp(join(tmpdir(), "fake-codex-"));
const events = [
{ type: "item.started", item: { id: "a", type: "command_execution", command: "mkdir f" } },
{ type: "item.started", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
{ type: "item.completed", item: { id: "b", type: "command_execution", command: "cat .agents/skills/s/SKILL.md" } },
{ type: "item.completed", item: { id: "a", type: "command_execution", command: "mkdir f" } },
{ type: "item.completed", item: { id: "c", type: "agent_message", text: "done" } },
{ type: "turn.completed", usage: { input_tokens: 10, output_tokens: 2 } },
];
const lines = events.map((event) => `printf '%s\\n' '${JSON.stringify(event)}'`).join("\n");
await writeFile(join(dir, "codex"), `#!/bin/sh\ncat > /dev/null\n${lines}\n`);
await chmod(join(dir, "codex"), 0o755);
try {
const result = await runCodex("p", dir, 12, "m", { PATH: `${dir}:${process.env.PATH}` });
expect(result.toolsUsed.map((tool) => tool.tool)).toEqual(["Bash", "Skill", "Bash"]);
expect(result.output).toBe("done");
expect(result.tokenUsage).toEqual({ prompt: 10, completion: 2, total: 12 });
} finally {
await rm(dir, { recursive: true, force: true });
}
});
});
ai_evals/adapters/cli/codex.ts
+241
View File
@@ -0,0 +1,241 @@
import { spawn } from "node:child_process";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createInterface } from "node:readline";
import type { BenchmarkTokenUsage, CliToolInvocation } from "../../core/types";
import { isLikelyMutatingBashCommand } from "./mutation";
export interface AgentRunResult {
output: string;
toolsUsed: CliToolInvocation[];
assistantMessageCount: number;
tokenUsage: BenchmarkTokenUsage | null;
finalContextTokens: number | null;
}
interface CodexItem {
type?: string;
text?: string;
command?: string;
changes?: { path?: string; kind?: string }[];
[key: string]: unknown;
}
interface CodexEvent {
type?: string;
item?: CodexItem;
usage?: { input_tokens?: number; output_tokens?: number };
message?: string;
error?: { message?: string };
}
const SKILL_FILE_PATTERN = /\.agents\/skills\/([^/\s'"`*?[]+)\/SKILL\.md/g;
// Drives `codex exec` and maps its events onto the Claude Code tool vocabulary
// the validators read: shell commands become `Bash`, patches `Edit`, and
// reading `.agents/skills/<name>/SKILL.md` (how Codex loads a skill) becomes a
// `Skill` call recorded just before that command.
export async function runCodex(
prompt: string,
cwd: string,
maxTurns: number,
model: string,
env: Record<string, string>
): Promise<AgentRunResult> {
// A fresh HOME keeps the user's ~/.codex config and ~/.agents/skills out of the run.
const home = await mkdtemp(join(tmpdir(), "wmill-cli-benchmark-codex-home-"));
const result: AgentRunResult = {
output: "",
toolsUsed: [],
assistantMessageCount: 0,
tokenUsage: null,
finalContextTokens: null,
};
const messages: string[] = [];
let failure: string | null = null;
let lastError: string | null = null;
let stderr = "";
try {
const child = spawn(
"codex",
[
"exec",
"--json",
"--ephemeral",
"--skip-git-repo-check",
"--dangerously-bypass-approvals-and-sandbox",
"--model",
model,
"--cd",
cwd,
"-",
],
{
cwd,
env: {
...env,
HOME: home,
CODEX_HOME: home,
CODEX_API_KEY: env.OPENAI_API_KEY ?? "",
},
stdio: ["pipe", "pipe", "pipe"],
}
);
// Without this, a missing `codex` binary leaves the stdout loop waiting forever.
await new Promise<void>((resolve, reject) => {
child.once("spawn", resolve);
child.once("error", reject);
});
child.stdin.on("error", () => {});
child.stdin.end(prompt);
child.stderr.on("data", (chunk) => {
stderr = (stderr + chunk).slice(-4000);
});
const exited = new Promise<number | null>((resolve, reject) => {
child.on("error", reject);
child.on("close", resolve);
});
// Codex has no turn cap. A turn starts when the agent issues tool calls with
// none in flight, so parallel calls count once, as they do for Claude Code.
// Claude Code's cap includes the final reply, so the maxTurns-th tool round is
// already over it.
// ponytail: parallel calls that finish before the next one starts count as
// separate turns; count model calls if Codex ever reports them.
let turns = 0;
let inFlight = 0;
let capped = false;
// Parallel calls complete out of order; the trace keeps the order they started in.
const slots = new Map<string, number>();
const ordered: CliToolInvocation[][] = [];
const slotFor = (item: CodexItem) => {
const id = typeof item.id === "string" ? item.id : undefined;
if (id !== undefined && slots.has(id)) {
return slots.get(id)!;
}
ordered.push([]);
if (id !== undefined) {
slots.set(id, ordered.length - 1);
}
return ordered.length - 1;
};
for await (const line of createInterface({ input: child.stdout })) {
const event = parseEvent(line);
if (!event) {
continue;
}
const item = event.item;
if (event.type === "item.started" && item && isToolItem(item)) {
if (inFlight === 0) {
turns += 1;
}
inFlight += 1;
slotFor(item);
if (turns >= maxTurns && !capped) {
capped = true;
child.kill("SIGTERM");
}
} else if (event.type === "item.completed" && item) {
if (item.type === "agent_message") {
result.assistantMessageCount += 1;
if (typeof item.text === "string") {
messages.push(item.text);
}
} else if (isToolItem(item)) {
inFlight = Math.max(0, inFlight - 1);
ordered[slotFor(item)] = toToolInvocations(item);
}
} else if (event.type === "turn.completed" && event.usage) {
const prompt = event.usage.input_tokens ?? 0;
const completion = event.usage.output_tokens ?? 0;
result.tokenUsage = { prompt, completion, total: prompt + completion };
} else if (event.type === "turn.failed") {
failure = event.error?.message ?? JSON.stringify(event);
} else if (event.type === "error") {
// Also sent for retries Codex recovers from ("Reconnecting... 1/5").
lastError = event.message ?? JSON.stringify(event);
}
}
result.toolsUsed = ordered.flat();
const code = await exited;
if (failure) {
throw new Error(`codex: ${failure}`);
}
// Fail the attempt, as Claude Code does at its cap.
if (capped) {
throw new Error(`codex: reached maximum number of turns (${maxTurns})`);
}
if (code !== 0) {
throw new Error(`codex exited with ${code}: ${lastError ?? stderr.trim()}`);
}
} finally {
await rm(home, { recursive: true, force: true });
}
result.output = messages.join("\n");
return result;
}
export function toToolInvocations(item: CodexItem): CliToolInvocation[] {
const timestamp = Date.now();
if (item.type === "command_execution" && typeof item.command === "string") {
const command = unwrapShellCommand(item.command);
// A skill read counts as preceding the command only when nothing before it in
// the command mutates; otherwise `printf x > f && cat SKILL.md` would pass a
// skill-before-first-mutation check.
const before: string[] = [];
const after: string[] = [];
for (const match of command.matchAll(SKILL_FILE_PATTERN)) {
const skill = match[1]!;
if (before.includes(skill) || after.includes(skill)) {
continue;
}
(isLikelyMutatingBashCommand(command.slice(0, match.index)) ? after : before).push(skill);
}
const skillCall = (skill: string) => ({ tool: "Skill", input: { skill }, timestamp });
return [
...before.map(skillCall),
{ tool: "Bash", input: { command }, timestamp },
...after.map(skillCall),
];
}
if (item.type === "file_change") {
return [{ tool: "Edit", input: { changes: item.changes ?? [] }, timestamp }];
}
const { id: _id, type, status: _status, ...input } = item;
return [{ tool: String(type), input, timestamp }];
}
// Codex reports commands as run through the user's login shell, e.g.
// `/usr/bin/zsh -lc 'cat SKILL.md'`.
export function unwrapShellCommand(command: string): string {
const match = command.match(/^(?:\S*\/)?(?:ba|z)?sh -l?c (['"])([\s\S]*)\1$/);
if (!match) {
return command;
}
const body = match[2]!;
return match[1] === "'"
? body.replaceAll(`'\\''`, "'").replaceAll(`'"'"'`, "'")
: body.replace(/\\(["\\$`])/g, "$1");
}
function isToolItem(item: CodexItem): boolean {
return !["agent_message", "reasoning", "todo_list", "error"].includes(item.type ?? "");
}
function parseEvent(line: string): CodexEvent | null {
try {
const value = JSON.parse(line);
return value && typeof value === "object" ? (value as CodexEvent) : null;
} catch {
return null;
}
}
ai_evals/adapters/cli/mutation.ts
+10
View File
@@ -0,0 +1,10 @@
export function isLikelyMutatingBashCommand(command: string): boolean {
return (
/\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
/\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
/\bsed\s+-i\b/.test(command) ||
/\bperl\s+-pi\b/.test(command) ||
// In command position only: `cat AGENTS.wmill.md` is a read.
/(?:^|[;&|(]|\b(?:then|do|xargs)\s)\s*wmill(?:\s|$)/m.test(command)
);
}
ai_evals/adapters/cli/runtime.test.ts
+31
View File
@@ -2,7 +2,9 @@ import { describe, expect, it } from "bun:test";
import {
anthropicUsageToBenchmarkTokenUsage,
extractCliResultTokenUsage,
endpointEnv,
extractProposedWmillCommands,
getFirstMutationToolIndex,
parseWmillInvocationLog,
} from "./runtime";
@@ -147,3 +149,32 @@ push
]);
});
});
describe("getFirstMutationToolIndex", () => {
it("counts wmill only as a command, not in a file name", () => {
const bash = (command: string) => ({ tool: "Bash", input: { command }, timestamp: 0 });
expect(
getFirstMutationToolIndex([
bash("cat AGENTS.wmill.md .agents/skills/x/SKILL.md"),
bash("ls && wmill generate-metadata"),
])
).toBe(1);
expect(getFirstMutationToolIndex([bash("cd f\nwmill sync push")])).toBe(0);
});
});
describe("endpointEnv", () => {
it("passes the endpoint only its own key", () => {
const env = endpointEnv(
{ baseUrl: "https://example.test", apiKeyEnv: "DEEPSEEK_API_KEY" },
"m",
{ PATH: "/bin", DEEPSEEK_API_KEY: "ds", OPENAI_API_KEY: "oa", ANTHROPIC_AUTH_TOKEN: "t" }
);
expect(env).toEqual({
PATH: "/bin",
ANTHROPIC_BASE_URL: "https://example.test",
ANTHROPIC_API_KEY: "ds",
ANTHROPIC_DEFAULT_HAIKU_MODEL: "m",
});
});
});
ai_evals/adapters/cli/runtime.ts
+103 -70
View File
@@ -2,7 +2,14 @@ import { query, type Options } from "@anthropic-ai/claude-agent-sdk";
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, join } from "path";
import { fileURLToPath } from "url";
import { getCliEvalModel, resolveEvalModel, type CliEvalModelConfig } from "../../core/models";
import {
getCliEvalModel,
resolveEvalModel,
type CliEvalModelConfig,
} from "../../core/models";
import { runCodex, type AgentRunResult } from "./codex";
import { isLikelyMutatingBashCommand } from "./mutation";
export { formatCliRunModelLabel } from "../../core/models";
import type {
BenchmarkTokenUsage,
CliToolInvocation,
@@ -141,16 +148,94 @@ export async function runPromptAndCapture(
maxTurns: number = 3,
modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
): Promise<PromptRunResult> {
const toolsUsed: ToolInvocation[] = [];
const skillsInvoked: string[] = [];
const bashCommands: string[] = [];
let output = "";
let assistantMessageCount = 0;
let tokenUsage: BenchmarkTokenUsage | null = null;
let finalContextTokens: number | null = null;
const startedAt = Date.now();
const stubBinDir = join(cwd, WMILL_STUB_DIR_NAME);
const wmillLogPath = join(cwd, WMILL_LOG_FILE_NAME);
const env = {
...getQueryEnv(),
PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
};
await installWmillStub(stubBinDir);
const run =
modelConfig.runtime === "codex"
? await runCodex(prompt, cwd, maxTurns, modelConfig.model, env)
: await runClaudeCode(prompt, cwd, maxTurns, modelConfig, env);
const skillsInvoked: string[] = [];
const bashCommands: string[] = [];
for (const tool of run.toolsUsed) {
if (tool.tool === "Skill" && typeof tool.input.skill === "string") {
pushUnique(skillsInvoked, tool.input.skill);
}
if (tool.tool === "Bash") {
for (const command of extractBashCommands(tool.input)) {
pushUnique(bashCommands, command);
}
}
}
const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
return {
output: run.output,
durationMs: Date.now() - startedAt,
tokenUsage: run.tokenUsage,
finalContextTokens: run.finalContextTokens,
trace: {
toolsUsed: run.toolsUsed,
skillsInvoked,
assistantMessageCount: run.assistantMessageCount,
bashCommands,
proposedCommands: extractProposedWmillCommands(run.output),
executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
wmillInvocations,
firstMutationToolIndex: getFirstMutationToolIndex(run.toolsUsed),
},
};
}
// Only the endpoint's own key may reach it: a missing key would make Claude Code
// fall back to the user's Claude login, and the agent's Bash inherits this env,
// so any other credential could end up in a tool result sent to the endpoint.
export function endpointEnv(
endpoint: NonNullable<CliEvalModelConfig["anthropicEndpoint"]>,
model: string,
env: Record<string, string>
): Record<string, string> {
const key = env[endpoint.apiKeyEnv];
if (!key) {
throw new Error(`${endpoint.apiKeyEnv} is not set`);
}
const rest = Object.fromEntries(
Object.entries(env).filter(([name]) => !/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL/i.test(name))
);
return {
...rest,
ANTHROPIC_BASE_URL: endpoint.baseUrl,
ANTHROPIC_API_KEY: key,
// Claude Code's background calls default to Haiku, which the endpoint does not serve.
ANTHROPIC_DEFAULT_HAIKU_MODEL: model,
};
}
async function runClaudeCode(
prompt: string,
cwd: string,
maxTurns: number,
modelConfig: CliEvalModelConfig,
env: Record<string, string>
): Promise<AgentRunResult> {
const result: AgentRunResult = {
output: "",
toolsUsed: [],
assistantMessageCount: 0,
tokenUsage: null,
finalContextTokens: null,
};
const endpoint = modelConfig.anthropicEndpoint;
const options: Options = {
cwd,
@@ -158,80 +243,42 @@ export async function runPromptAndCapture(
maxTurns,
settingSources: ["project"],
allowedTools: ["Skill", "Read", "Glob", "Grep", "Bash", "Write", "Edit"],
env: {
...getQueryEnv(),
PATH: process.env.PATH ? `${stubBinDir}${delimiter}${process.env.PATH}` : stubBinDir,
WMILL_BENCHMARK_LOG_PATH: wmillLogPath,
},
env: endpoint ? endpointEnv(endpoint, modelConfig.model, env) : env,
};
await installWmillStub(stubBinDir);
for await (const message of query({ prompt, options })) {
if (message.type === "assistant") {
assistantMessageCount += 1;
result.assistantMessageCount += 1;
const turnContext = anthropicUsageToBenchmarkTokenUsage(
message.message?.usage
)?.prompt;
if (turnContext && turnContext > 0) {
finalContextTokens = turnContext;
result.finalContextTokens = turnContext;
}
const content = message.message?.content;
if (Array.isArray(content)) {
for (const block of content) {
if (block.type === "tool_use") {
const input = normalizeToolInput(block.input);
toolsUsed.push({
result.toolsUsed.push({
tool: block.name,
input,
input: normalizeToolInput(block.input),
timestamp: Date.now()
});
if (block.name === "Skill") {
const skillInput = input as { skill?: string };
if (skillInput.skill) {
pushUnique(skillsInvoked, skillInput.skill);
}
}
if (block.name === "Bash") {
for (const command of extractBashCommands(input)) {
pushUnique(bashCommands, command);
}
}
} else if (block.type === "text") {
output += block.text;
result.output += block.text;
}
}
}
} else if (message.type === "result") {
const resultMessage = message as { result?: string };
tokenUsage = extractCliResultTokenUsage(message) ?? tokenUsage;
result.tokenUsage = extractCliResultTokenUsage(message) ?? result.tokenUsage;
if (typeof resultMessage.result === "string") {
output += resultMessage.result;
result.output += resultMessage.result;
}
}
}
const proposedCommands = extractProposedWmillCommands(output);
const wmillInvocations = await readWmillInvocationLog(wmillLogPath);
return {
output,
durationMs: Date.now() - startedAt,
tokenUsage,
finalContextTokens,
trace: {
toolsUsed,
skillsInvoked,
assistantMessageCount,
bashCommands,
proposedCommands,
executedWmillCommands: wmillInvocations.map(formatExecutedWmillCommand),
wmillInvocations,
firstMutationToolIndex: getFirstMutationToolIndex(toolsUsed),
},
};
return result;
}
export function wasSkillInvoked(result: PromptRunResult, skillName: string): boolean {
@@ -242,10 +289,6 @@ export function wasToolUsed(result: PromptRunResult, toolName: string): boolean
return result.trace.toolsUsed.some((tool) => tool.tool === toolName);
}
export function formatCliRunModelLabel(modelConfig: CliEvalModelConfig): string {
return `${modelConfig.provider}:${modelConfig.model}`;
}
export function getToolInputs(
result: PromptRunResult,
toolName: string
@@ -512,7 +555,7 @@ function formatExecutedWmillCommand(entry: CliWmillInvocation): string {
return ["wmill", ...entry.argv].join(" ").trim();
}
function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
export function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
for (const [index, tool] of toolsUsed.entries()) {
if (tool.tool === "Write" || tool.tool === "Edit") {
return index;
@@ -526,16 +569,6 @@ function getFirstMutationToolIndex(toolsUsed: ToolInvocation[]): number | null {
return null;
}
function isLikelyMutatingBashCommand(command: string): boolean {
return (
/\b(?:mkdir|touch|rm|mv|cp|install|tee)\b/.test(command) ||
/\b(?:cat|echo|printf)\b.*(?:>|>>|\|\s*tee\b)/.test(command) ||
/\bsed\s+-i\b/.test(command) ||
/\bperl\s+-pi\b/.test(command) ||
/\bwmill\b/.test(command)
);
}
function pushUnique(values: string[], value: string): void {
if (!values.includes(value)) {
values.push(value);
ai_evals/adapters/frontend/core/global/globalEvalRunner.ts
+2
View File
@@ -123,6 +123,7 @@ export interface GlobalEvalOptions {
planMode?: boolean;
model?: string;
maxIterations?: number;
userAnswers?: string[];
provider?: AIProvider;
backend: WindmillBackendSettings;
workspaceRoot?: string;
@@ -220,6 +221,7 @@ export async function runGlobalEval(
onToolCall: options.runContext?.onToolCall,
options: {
maxIterations: options.maxIterations,
userAnswers: options.userAnswers,
model,
workspace: workspaceRoot,
provider: options.provider,
ai_evals/adapters/frontend/core/shared/baseEvalRunner.ts
+7
View File
@@ -85,6 +85,7 @@ export async function runEval<THelpers, TOutput>(
const model = options.model ?? "gpt-4o";
const maxIterations = options.maxIterations ?? 20;
const pendingAnswers = options.userAnswers && [...options.userAnswers];
const workspace = options.workspace ?? "test-workspace";
const provider = toFrontendEvalProvider(options.provider);
@@ -138,6 +139,12 @@ export async function runEval<THelpers, TOutput>(
// edit the arguments, so a case can assert what the model proposed but never how
// it reacts to the user changing something.
requestRunArgs: async (_toolId, form) => form.args,
...(pendingAnswers && {
requestUserQuestion: async () => {
const answer = pendingAnswers.shift();
return answer === undefined ? undefined : [answer];
},
}),
onNewToken: (token: string) => {
if (shouldEmitMessageStart) {
onAssistantMessageStart?.();
ai_evals/adapters/frontend/core/shared/types.ts
+3
View File
@@ -16,6 +16,9 @@ export interface ToolCallDetail {
export interface EvalRunnerOptions {
backend: WindmillBackendSettings;
maxIterations?: number;
// Scripted answers to the model's questions, one per question in order; a question past
// the last one is dismissed. Unset, the run cannot be asked anything.
userAnswers?: string[];
model?: string;
workspace?: string;
provider?: AIProvider;
ai_evals/adapters/frontend/vitestAdapter.test.ts
+10 -2
View File
@@ -6,6 +6,8 @@ import { dirname, resolve } from 'path'
// @ts-ignore - Node.js url
import { fileURLToPath } from 'url'
import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend'
import { resolveWindmillBackendSettings } from '../../core/windmillBackendSettings'
import { WindmillBackendClient } from './windmillBackend'
const FRONTEND_DIR = fileURLToPath(new URL('../../../frontend/', import.meta.url))
@@ -28,6 +30,11 @@ globalThis.fetch = (async (input: unknown, init?: RequestInit) => {
if (typeof url === 'string' && hasBenchmarkApiHandler(url)) {
return handleBenchmarkApiFetch(url, init)
}
// The docs tools read the documentation bundled into the backend: hand them to the
// benchmark's real backend, logged in like the AI proxy calls.
if (typeof url === 'string' && url.startsWith('/api/docs/')) {
return new WindmillBackendClient(resolveWindmillBackendSettings()).request(url.slice('/api'.length), init)
}
// The parsers behind inferArgs load their wasm from a vite `?url` path, which only a dev
// server serves. Unserved, every script schema infers as empty, and the tools then tell the
// model its arguments are undeclared, so it rewrites a correct script until it runs out of turns.
@@ -684,6 +691,7 @@ benchmarkIt(
resetBenchmarkMockBackend()
}
},
// Full-suite runs (30+ cases at concurrency 2-3) routinely exceed 10 minutes.
7_200_000
// A full suite on a slow reasoning model runs for hours (gpt-6-astra on global went past
// 2h); the CI job's own timeout is the real bound.
18_000_000
)
ai_evals/cases/global.yaml
+106 -1
View File
@@ -198,22 +198,127 @@
- uses the requested placeholder value and description
- does not create a resource or deploy anything
# A new app that has to store data, with nothing said about where: the data setup is the
# user's call, so it is asked before anything is created. The question is dismissed here,
# which leaves nothing the model may build on.
- id: global-test7-ambiguous-app-asks-question
prompt: |-
Create a new raw app for triaging support tickets.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 4
maxTurns: 6
userAnswers: []
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- list_datatables
- askUserQuestion
forbiddenToolsUsed:
- init_app
- exec_datatable_sql
- write_app_file
- write_app_runnable
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: askUserQuestion
field: question
stringIncludesAnyOf:
- data table
- datatable
- schema
- tables
- store
skipJudge: true
# The same request with the question answered: the app is built on what the user chose.
- id: global-new-app-data-setup-answer-applied
prompt: |-
Create a new raw app for triaging support tickets.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 32
userAnswers:
- A new schema named helpdesk
- Create new tables
validate:
draftCountExactly: 1
requiredDrafts:
- type: app
valueIncludes:
- helpdesk
- react-dom/client
toolExpect:
requiredToolsUsed:
- askUserQuestion
- init_app
forbiddenToolsUsed:
- deploy_workspace_item
toolCallArgs:
- tool: init_app
field: framework
stringEqualsAnyOf:
- react19
- tool: exec_datatable_sql
field: sql
stringIncludesAnyOf:
- helpdesk
skipJudge: true
# No framework named, no app to follow, nothing to store: the modal's recommended
# framework is used without a question.
- id: global-new-app-defaults-to-react19
prompt: |-
Build me a little app that converts temperatures between Celsius and Fahrenheit.
runtime:
maxTurns: 12
validate:
draftCountExactly: 1
requiredDrafts:
- type: app
valueIncludes:
- react-dom/client
toolExpect:
requiredToolsUsed:
- init_app
forbiddenToolsUsed:
- askUserQuestion
- exec_datatable_sql
- deploy_workspace_item
toolCallArgs:
- tool: init_app
field: framework
stringEqualsAnyOf:
- react19
skipJudge: true
# The user said where the data goes, so there is nothing left to ask.
- id: global-new-app-storage-specified-no-question
prompt: |-
Build a todo list app in React. Keep the todos in the main data table, in a new
schema called todo_app.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 20
validate:
draftCountExactly: 1
requiredDrafts:
- type: app
valueIncludes:
- todo_app
toolExpect:
requiredToolsUsed:
- init_app
- write_app_runnable
forbiddenToolsUsed:
- askUserQuestion
- deploy_workspace_item
toolCallArgs:
- tool: exec_datatable_sql
field: sql
stringIncludesAnyOf:
- todo_app
skipJudge: true
- id: global-test8-human-script-infer-path-language
ai_evals/core/cases.test.ts
-2
View File
@@ -192,8 +192,6 @@ describe("loadCases", () => {
requiredSkills: ["write-script-bun"],
requiredSkillsBeforeFirstMutation: ["write-script-bun"],
forbiddenSkills: ["write-script-python3", "write-flow"],
orderedAssistantMentions: ["wmill generate-metadata", "wmill sync push"],
orderedProposedCommands: ["wmill generate-metadata", "wmill sync push"],
forbiddenExecutedCommands: ["^wmill generate-metadata", "^wmill sync push"],
});
});
ai_evals/core/models.test.ts
+13 -1
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "bun:test";
import { resolveEvalModel } from "./models";
import { formatRunModelLabel, resolveEvalModel } from "./models";
describe("resolveEvalModel", () => {
it("supports GPT-5.5 aliases for frontend evals", () => {
@@ -47,6 +47,18 @@ describe("resolveEvalModel", () => {
});
});
it("labels cli runs by agent, keeping Claude's historical label", () => {
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "sonnet-5.5"))).toBe(
"anthropic:claude-sonnet-5-5",
);
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "gpt-6.1-sol"))).toBe(
"codex:gpt-6.1-sol",
);
expect(formatRunModelLabel("cli", resolveEvalModel("cli", "deepseek-v4-pro"))).toBe(
"claude-code:deepseek-v4-pro",
);
});
it("rejects Gemini aliases for cli evals", () => {
expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow(
"Model gemini-3-flash-preview is not supported for cli mode",
ai_evals/core/models.ts
+45 -7
View File
@@ -5,9 +5,13 @@ export interface FrontendEvalModelConfig {
model: string;
}
// The coding agent that drives the model in cli mode (adapters/cli/runtime.ts).
export interface CliEvalModelConfig {
provider: "anthropic";
runtime: "claude-code" | "codex";
model: string;
// An Anthropic-compatible endpoint other than Anthropic's, for claude-code,
// and the env var holding its key.
anthropicEndpoint?: { baseUrl: string; apiKeyEnv: string };
}
export interface EvalModelSpec {
@@ -35,7 +39,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "claude-haiku-4-5-20251001",
},
cli: {
provider: "anthropic",
runtime: "claude-code",
model: "haiku",
},
},
@@ -55,7 +59,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "claude-sonnet-4-5-20250929",
},
cli: {
provider: "anthropic",
runtime: "claude-code",
model: "sonnet",
},
},
@@ -74,7 +78,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "claude-opus-4-6",
},
cli: {
provider: "anthropic",
runtime: "claude-code",
model: "opus",
},
},
@@ -87,7 +91,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "claude-sonnet-5-5",
},
cli: {
provider: "anthropic",
runtime: "claude-code",
model: "claude-sonnet-5-5",
},
},
@@ -100,7 +104,7 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "claude-opus-5-5",
},
cli: {
provider: "anthropic",
runtime: "claude-code",
model: "claude-opus-5-5",
},
},
@@ -139,6 +143,10 @@ export const EVAL_MODELS: EvalModelSpec[] = [
provider: "openai",
model: "gpt-6-astra",
},
cli: {
runtime: "codex",
model: "gpt-6-astra",
},
},
{
id: "gpt-6-sol",
@@ -149,6 +157,19 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "gpt-6-sol",
},
},
{
id: "gpt-6.1-sol",
label: "GPT-6.1 Sol",
aliases: ["gpt-6.1-sol"],
frontend: {
provider: "openai",
model: "gpt-6.1-sol",
},
cli: {
runtime: "codex",
model: "gpt-6.1-sol",
},
},
{
id: "gpt-6-luna",
label: "GPT-6 Luna",
@@ -215,6 +236,14 @@ export const EVAL_MODELS: EvalModelSpec[] = [
provider: "deepseek",
model: "deepseek-v4-pro",
},
cli: {
runtime: "claude-code",
model: "deepseek-v4-pro",
anthropicEndpoint: {
baseUrl: "https://api.deepseek.com/anthropic",
apiKeyEnv: "DEEPSEEK_API_KEY",
},
},
},
];
@@ -253,11 +282,20 @@ export function formatRunModelLabel(
model: EvalModelSpec,
): string {
if (mode === "cli") {
return `${model.cli!.provider}:${model.cli!.model}`;
return formatCliRunModelLabel(model.cli!);
}
return `${model.frontend!.provider}:${model.frontend!.model}`;
}
// Claude on Claude Code keeps the `anthropic:` prefix its dashboard history was
// recorded under; any other pairing names the agent.
export function formatCliRunModelLabel(config: CliEvalModelConfig): string {
if (config.runtime === "claude-code" && !config.anthropicEndpoint) {
return `anthropic:${config.model}`;
}
return `${config.runtime}:${config.model}`;
}
export function getFrontendEvalModel(
model: EvalModelSpec,
): FrontendEvalModelConfig {
ai_evals/core/types.ts
+4
View File
@@ -29,6 +29,10 @@ export interface EvalCaseRuntimeAppContextSpec {
export interface EvalCaseRuntimeSpec {
maxTurns?: number;
// Global mode: scripted answers to askUserQuestion, one per question in order; a question
// past the last one is dismissed, so `[]` dismisses them all. Unset, the question tool
// reports that it cannot ask.
userAnswers?: string[];
backendPreview?: EvalCaseRuntimeBackendPreview;
appContext?: EvalCaseRuntimeAppContextSpec;
// Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat.
ai_evals/modes/cli.ts
+32 -9
View File
@@ -39,17 +39,36 @@ interface CliRunActual {
trace: CliTrace;
}
const CLAUDE_PROJECT_PREAMBLE = [
"Follow the project instructions from AGENTS.md exactly.",
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
"Do not skip the Skill step.",
// Only the skill wording differs per agent: Claude Code has a Skill tool, Codex
// loads a skill by reading its SKILL.md.
const SKILL_STEP: Record<CliEvalModelConfig["runtime"], string[]> = {
"claude-code": [
"Before creating or modifying any Windmill entity, you MUST invoke the relevant Skill tool and follow it.",
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
"Do not skip the Skill step.",
],
codex: [
"Before creating or modifying any Windmill entity, you MUST read the relevant skill's SKILL.md under .agents/skills and follow it.",
"Use the skill guidance for file layout, implementation details, and the exact next commands to tell the user.",
"Do not skip the skill step.",
],
};
const HARNESS_PREAMBLE = [
"You are running inside an automated benchmark harness, not an interactive user session.",
"Act autonomously and complete the requested file changes directly in the workspace.",
"Do not ask for confirmation, do not ask the user to save or create files manually, and do not wait for approval.",
"Do not respond with a plan when you can make the change directly.",
"Only describe what was done after you have written the files.",
].join(" ");
];
function projectPreamble(runtime: CliEvalModelConfig["runtime"]): string {
return [
"Follow the project instructions from AGENTS.md exactly.",
...SKILL_STEP[runtime],
...HARNESS_PREAMBLE,
].join(" ");
}
export function createCliModeRunner(
modelConfig: CliEvalModelConfig = DEFAULT_CLI_EVAL_MODEL
@@ -89,7 +108,7 @@ export function createCliModeRunner(
});
await writeFile(join(workspaceDir, "rt.d.ts"), "export namespace RT {}\n", "utf8");
const renderedPrompt = await renderPrompt(prompt, workspaceDir);
const renderedPrompt = await renderPrompt(prompt, workspaceDir, modelConfig.runtime);
const run = await runPromptAndCapture(
renderedPrompt,
workspaceDir,
@@ -193,7 +212,11 @@ export function getCliRunModelLabel(
return formatCliRunModelLabel(modelConfig);
}
async function renderPrompt(prompt: string, workspaceDir: string): Promise<string> {
async function renderPrompt(
prompt: string,
workspaceDir: string,
runtime: CliEvalModelConfig["runtime"]
): Promise<string> {
const renderedUserPrompt = prompt.replaceAll("{{workspace_root}}", workspaceDir);
const agentsInstructions = await readFile(path.join(workspaceDir, "AGENTS.md"), "utf8");
@@ -202,7 +225,7 @@ async function renderPrompt(prompt: string, workspaceDir: string): Promise<strin
agentsInstructions.trim(),
"",
"# Benchmark Harness",
CLAUDE_PROJECT_PREAMBLE,
projectPreamble(runtime),
"",
"# User Request",
renderedUserPrompt,
ai_evals/modes/global.ts
+1
View File
@@ -54,6 +54,7 @@ export function createGlobalModeRunner(
sessionChat: context.evalCase?.runtime?.sessionChat,
planMode: context.evalCase?.runtime?.planMode,
maxIterations: context.evalCase?.runtime?.maxTurns,
userAnswers: context.evalCase?.runtime?.userAnswers,
provider: modelConfig.provider,
model: modelConfig.model,
backend: backendSettings,
backend/.sqlx/query-03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\nINSERT INTO concurrency_counter(concurrency_id, job_uuids)\nVALUES ($1, '{}'::jsonb)\nON CONFLICT (concurrency_id)\nDO UPDATE SET job_uuids = concurrency_counter.job_uuids\nRETURNING (SELECT COUNT(*)\n FROM jsonb_object_keys(job_uuids) AS keys(key)\n WHERE key <> $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": [
null
]
},
"hash": "03cc00b0b5b8b1ffcc14ac5f6cb3062a8f706e70bc9c4b3764cd1d6f99b9440a"
}
backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json → backend/.sqlx/query-d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1",
"query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\",\n COUNT(*) FILTER (WHERE is_service_account AND NOT disabled) AS \"service_accounts!\"\n FROM usr WHERE workspace_id = $1",
"describe": {
"columns": [
{
@@ -12,6 +12,11 @@
"ordinal": 1,
"name": "operators!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "service_accounts!",
"type_info": "Int8"
}
],
"parameters": {
@@ -20,9 +25,10 @@
]
},
"nullable": [
null,
null,
null
]
},
"hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b"
"hash": "d61e8523269a26051079b0ee837bf33c1ce2e063aaf80fd155dbbbf99eb2fab3"
}
backend/Cargo.lock
Generated
+151 -149
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.823.1"
version = "1.826.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.823.1"
version = "1.826.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
8637aa03cbe1534a9d1a8efe1fc15e27fc788cfd
72a86aa16e892ad094d9d1c3971eb20272ebf824
backend/oauth_connect.json
+47 -5
View File
@@ -45,12 +45,39 @@
"files:read"
],
"scope_options": [
{ "value": "chat:write", "label": "Send messages as the app" },
{ "value": "chat:write.public", "label": "Send messages to public channels the app is not in" },
{ "value": "channels:join", "label": "Join public channels" },
{ "value": "files:write", "label": "Upload, edit and delete files" },
{ "value": "users:read", "label": "View people in the workspace" },
{ "value": "users:read.email", "label": "View people's email addresses" },
{ "value": "channels:read", "label": "View basic information about public channels" },
{ "value": "channels:history", "label": "Read messages in public channels the app is in" },
{ "value": "reactions:write", "label": "Add emoji reactions" },
{ "value": "channels:manage", "label": "Create, archive and manage public channels, invite members, set topics" },
{ "value": "groups:write", "label": "Create and manage private channels the app is in" },
{ "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" }
{ "value": "usergroups:write", "label": "Edit user groups (workspace settings often block bots)" },
{ "value": "files:read", "label": "Read and download files shared where the app is a member" },
{ "value": "groups:read", "label": "List private channels the app is in" },
{ "value": "im:read", "label": "List the app's direct messages" },
{ "value": "mpim:read", "label": "List the app's group direct messages" },
{ "value": "usergroups:read", "label": "List user groups" },
{ "value": "emoji:read", "label": "List custom emojis" }
],
"user_scope_options": [
{ "value": "channels:history", "label": "Read messages in public channels" },
{ "value": "groups:history", "label": "Read messages in your private channels" },
{ "value": "im:history", "label": "Read your direct messages" },
{ "value": "mpim:history", "label": "Read your group direct messages" },
{ "value": "search:read", "label": "Search messages and files" },
{ "value": "users:read", "label": "View people in the workspace" },
{ "value": "chat:write", "label": "Send messages as you" },
{ "value": "users:read.email", "label": "View people's email addresses" },
{ "value": "channels:read", "label": "View basic information about public channels" },
{ "value": "groups:read", "label": "View basic information about your private channels" },
{ "value": "im:read", "label": "View basic information about your direct messages" },
{ "value": "mpim:read", "label": "View basic information about your group direct messages" },
{ "value": "files:read", "label": "View files shared in your conversations" },
{ "value": "reactions:write", "label": "Add emoji reactions" },
{ "value": "channels:write", "label": "Create, archive and manage public channels" },
{ "value": "channels:write.invites", "label": "Invite members to public channels" },
@@ -58,7 +85,10 @@
{ "value": "groups:write", "label": "Create and manage private channels" },
{ "value": "users.profile:write", "label": "Set your status and edit your profile" },
{ "value": "reminders:write", "label": "Create reminders for yourself" },
{ "value": "usergroups:write", "label": "Edit user groups" }
{ "value": "usergroups:write", "label": "Edit user groups" },
{ "value": "files:write", "label": "Upload and delete files as yourself" },
{ "value": "usergroups:read", "label": "List user groups" },
{ "value": "emoji:read", "label": "List custom emojis" }
],
"token_response_path": "authed_user"
},
@@ -73,7 +103,8 @@
"scopes": ["https://www.googleapis.com/auth/spreadsheets"],
"scope_options": [
{ "value": "https://www.googleapis.com/auth/spreadsheets", "label": "Read, edit, create and delete spreadsheets" },
{ "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" }
{ "value": "https://www.googleapis.com/auth/spreadsheets.readonly", "label": "Read spreadsheets" },
{ "value": "https://www.googleapis.com/auth/drive.metadata.readonly", "label": "List and search your spreadsheets in Drive" }
],
"extra_params": {
"access_type": "offline",
@@ -150,7 +181,9 @@
"scope_options": [
{ "value": "https://www.googleapis.com/auth/documents", "label": "Read, edit, create and delete documents" },
{ "value": "https://www.googleapis.com/auth/documents.readonly", "label": "Read documents" },
{ "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" }
{ "value": "https://www.googleapis.com/auth/drive.file", "label": "Only files created or opened with this connection" },
{ "value": "https://www.googleapis.com/auth/drive.readonly", "label": "Read, export and search all Drive files and comments" },
{ "value": "https://www.googleapis.com/auth/drive", "label": "Read, edit, create and delete all Drive files" }
],
"extra_params": {
"access_type": "offline",
@@ -205,7 +238,8 @@
{ "value": "https://www.googleapis.com/auth/admin.directory.group.readonly", "label": "Read groups" },
{ "value": "https://www.googleapis.com/auth/admin.directory.orgunit", "label": "Read and manage organizational units" },
{ "value": "https://www.googleapis.com/auth/admin.directory.orgunit.readonly", "label": "Read organizational units" },
{ "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" }
{ "value": "https://www.googleapis.com/auth/admin.directory.user.security", "label": "Manage user security settings" },
{ "value": "https://www.googleapis.com/auth/admin.reports.audit.readonly", "label": "Read audit logs (login, admin, Drive, token activity)" }
],
"extra_params": {
"access_type": "offline",
@@ -220,6 +254,14 @@
"type": "web_server"
}
},
"notion": {
"auth_url": "https://api.notion.com/v1/oauth/authorize",
"token_url": "https://api.notion.com/v1/oauth/token",
"scopes": [],
"extra_params": {
"owner": "user"
}
},
"linkedin": {
"auth_url": "https://www.linkedin.com/oauth/v2/authorization",
"token_url": "https://www.linkedin.com/oauth/v2/accessToken",
backend/parsers/windmill-parser-sql/src/lib.rs
+7 -5
View File
@@ -1040,11 +1040,13 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ {
pub fn parse_snowflake_typ(typ: &str) -> Typ {
match typ {
"varchar" => Typ::Str(None),
"binary" => Typ::Bytes,
"date" | "time" | "timestamp" => Typ::Datetime,
"int" => Typ::Int,
"float" => Typ::Float,
"varchar" | "text" | "string" | "char" | "character" => Typ::Str(None),
"binary" | "varbinary" => Typ::Bytes,
"date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz"
| "timestamp_tz" => Typ::Datetime,
"int" | "integer" | "bigint" | "smallint" | "tinyint" | "byteint" => Typ::Int,
"float" | "float4" | "float8" | "double" | "double precision" | "real" | "number"
| "decimal" | "numeric" => Typ::Float,
"boolean" => Typ::Bool,
"s3object" => Typ::Resource("S3Object".to_string()),
_ => Typ::Str(None),
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6302,7 +6302,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6314,7 +6314,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"convert_case",
"serde",
@@ -6323,7 +6323,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6335,7 +6335,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6347,7 +6347,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6359,7 +6359,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6371,7 +6371,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6383,7 +6383,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6394,7 +6394,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6405,7 +6405,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6417,7 +6417,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6428,7 +6428,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6450,7 +6450,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6462,7 +6462,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6493,7 +6493,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"serde",
@@ -6518,7 +6518,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6536,7 +6536,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6552,7 +6552,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6568,7 +6568,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6600,7 +6600,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6614,7 +6614,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.823.1"
version = "1.826.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.823.1"
version = "1.826.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/src/monitor.rs
+13 -1
View File
@@ -40,7 +40,7 @@ use windmill_common::ee_oss::{jobs_waiting_alerts, worker_groups_alerts};
use windmill_common::global_settings::OAUTH_SETTING;
use windmill_common::otel_oss::{
otel_incr_zombie_delete_count, otel_incr_zombie_restart_count, otel_set_db_pool,
otel_set_queue_count, otel_set_queue_running_count,
otel_set_queue_count, otel_set_queue_delay, otel_set_queue_running_count,
};
use windmill_common::{
agent_workers::DECODED_AGENT_TOKEN,
@@ -168,6 +168,12 @@ lazy_static::lazy_static! {
&["tag"]
).unwrap();
static ref QUEUE_DELAY_SECONDS: prometheus::GaugeVec = prometheus::register_gauge_vec!(
"queue_delay_seconds",
"How long the next job to be picked up has been waiting, in seconds",
&["tag"]
).unwrap();
static ref QUEUE_RUNNING_COUNT: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
"queue_running_count",
"Number of running jobs in the queue",
@@ -5597,6 +5603,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
for q in QUEUE_COUNT_TAGS.read().await.iter() {
if queue_stats.get(q).is_none() {
(*QUEUE_COUNT).with_label_values(&[q]).set(0);
(*QUEUE_DELAY_SECONDS).with_label_values(&[q]).set(0.0);
}
}
}
@@ -5605,6 +5612,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
for q in OTEL_QUEUE_COUNT_TAGS.read().await.iter() {
if queue_stats.get(q).is_none() {
otel_set_queue_count(q, 0);
otel_set_queue_delay(q, 0.0);
}
}
}
@@ -5620,6 +5628,9 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
if metrics_enabled {
let metric = (*QUEUE_COUNT).with_label_values(&[tag]);
metric.set(count as i64);
(*QUEUE_DELAY_SECONDS)
.with_label_values(&[tag])
.set(stat.delay);
tags_to_watch.push(tag.to_string());
}
@@ -5627,6 +5638,7 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
otel_tags_to_watch.push(tag.to_string());
}
otel_set_queue_count(tag, count as i64);
otel_set_queue_delay(tag, stat.delay);
}
if save_metrics {
backend/tests/ai_agent_nul.rs
+125
View File
@@ -0,0 +1,125 @@
use axum::{routing::post, Json, Router};
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use std::time::Duration;
use windmill_common::{flows::FlowValue, jobs::JobPayload};
use windmill_test_utils::{
completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind,
};
const MEMORY_ID: &str = "0199c0de-0000-7000-8000-000000000001";
/// A model that puts a U+0000 in its tool call arguments, then in its answer.
async fn model(Json(body): Json<Value>) -> ([(&'static str, &'static str); 1], String) {
let finished = body["messages"]
.as_array()
.unwrap()
.iter()
.any(|m| m["role"] == "tool");
let delta = if finished {
json!({"role":"assistant", "content":"are\u{0}tired"})
} else {
json!({"role":"assistant", "tool_calls":[{
"index":0, "id":"call_0", "type":"function",
"function":{"name":"echo", "arguments":"{\"x\":\"a\\u0000b\"}"}
}]})
};
let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]});
let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]});
(
[("content-type", "text/event-stream")],
format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"),
)
}
async fn run_flow(db: &Pool<Postgres>, modules: Value) -> anyhow::Result<Value> {
let server = ApiServer::start(db.clone()).await?;
let flow: FlowValue = serde_json::from_value(json!({ "modules": modules }))?;
let id = RunJob::from(JobPayload::RawFlow {
value: flow,
path: Some("u/test/nul".into()),
restarted_from: None,
})
.push(db)
.await;
let notifications = listen_for_completed_jobs(db).await;
tokio::time::timeout(
Duration::from_secs(45),
in_test_worker(db, notifications.find(&id), server.addr.port()),
)
.await?;
let job = completed_job(id, db).await;
server.close().await?;
assert!(job.success, "{:?}", job.result);
Ok(job.json_result().expect("flow result"))
}
#[sqlx::test(fixtures("base"))]
#[serial_test::serial]
async fn agent_step_survives_nul_in_model_output(db: Pool<Postgres>) -> anyhow::Result<()> {
std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true");
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let base = format!("http://{}", listener.local_addr()?);
let stub = tokio::spawn(async move {
let router = Router::new().route("/v1/chat/completions", post(model));
axum::serve(listener, router).await.unwrap();
});
let result = run_flow(
&db,
json!([{"id":"agent","value":{
"type":"aiagent",
"tools":[{"id":"t0","summary":"echo","value":{
"type":"rawscript", "language":"bun", "input_transforms":{},
"content":"export async function main(x: string) { return x }"
}}],
"input_transforms":{
"provider":{"type":"static","value":{"kind":"customai","model":"nul-test","resource":{"base_url":format!("{base}/v1")}}},
"user_message":{"type":"static","value":"echo"},
"memory":{"type":"static","value":{"kind":"auto","context_length":10,"memory_id":MEMORY_ID}}
}
}}]),
)
.await?;
stub.abort();
assert_eq!(result["output"], "aretired");
let tool_message = result["messages"]
.as_array()
.unwrap()
.iter()
.find(|m| m["role"] == "tool")
.expect("tool message");
assert_eq!(tool_message["content"], "\"ab\"");
// The model's own words are also written to a jsonb column the job result never passes through.
let memory: String = sqlx::query_scalar(
"SELECT messages::text FROM ai_agent_memory WHERE conversation_id = $1::uuid",
)
.bind(MEMORY_ID)
.fetch_one(&db)
.await?;
assert!(memory.contains("aretired"), "{memory}");
Ok(())
}
/// A step's args are evaluated from the previous step's result as the worker holds it, not as
/// the completed job stored it.
#[sqlx::test(fixtures("base"))]
#[serial_test::serial]
async fn flow_step_args_survive_nul_in_previous_result(db: Pool<Postgres>) -> anyhow::Result<()> {
let result = run_flow(
&db,
json!([
{"id":"a","value":{
"type":"rawscript", "language":"bun", "input_transforms":{},
"content":"export async function main() { return 'a\\u0000b' }"
}},
{"id":"b","value":{
"type":"rawscript", "language":"bun",
"input_transforms":{"x":{"type":"javascript","expr":"results.a"}},
"content":"export async function main(x: string) { return x }"
}}
]),
)
.await?;
assert_eq!(result, "ab");
Ok(())
}
backend/tests/ai_decision.rs
+148
View File
@@ -0,0 +1,148 @@
//! AI decision steps run end to end against a stand-in for the decision endpoints: TypeSafe's
//! System One, Cloudflare Workers AI's, which serves each model at its own URL inside an API
//! envelope, and OpenAI's Decisions API.
use axum::{routing::post, Json, Router};
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use windmill_common::{flows::FlowValue, jobs::JobPayload};
use windmill_test_utils::*;
/// Answers every question as a choice of `refund`, the way TypeSafe shapes an answer.
fn refund_answers(body: &Value) -> Value {
let answers = body["questions"]
.as_object()
.into_iter()
.flatten()
.map(|(name, _)| {
let answer = json!({
"type": "choice",
"choice": "refund",
"probabilities": {"refund": 0.9, "bug": 0.1},
"confidence": 0.8
});
(name.clone(), answer)
})
.collect::<serde_json::Map<_, _>>();
json!({"model": body["model"], "answers": answers, "usage": {"input_tokens": 10}})
}
async fn start_decision_stub() -> anyhow::Result<u16> {
let app = Router::new()
.route(
"/v1/systemone",
post(|Json(body): Json<Value>| async move { Json(refund_answers(&body)) }),
)
.route(
"/accounts/acc/ai/run/@cf/cloudflare/clef-flash",
post(|Json(body): Json<Value>| async move {
Json(json!({"result": refund_answers(&body), "success": true, "errors": []}))
}),
)
// OpenAI's Decisions API takes the questions as a list and answers with one.
.route(
"/openai/v1/decisions",
post(|Json(body): Json<Value>| async move {
let answers = body["questions"]
.as_array()
.into_iter()
.flatten()
.map(|q| json!({"type": "choice", "name": q["name"], "choice": "refund"}))
.collect::<Vec<_>>();
Json(json!({"answers": answers}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let port = listener.local_addr()?.port();
tokio::spawn(async move { axum::serve(listener, app).await });
Ok(port)
}
/// A decision's answers reach later steps as `results.<id>.output`, which a Branch to one after it
/// routes on.
#[cfg(feature = "quickjs")]
#[sqlx::test(fixtures("base"))]
async fn test_ai_decision_answers_route_a_following_branch(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
// The stub listens on loopback, which the SSRF check refuses without this.
std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true");
let stub_port = start_decision_stub().await?;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
for provider in [
json!({
"kind": "typesafe",
"resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/v1")},
"model": "jev-latest"
}),
json!({
"kind": "cloudflare",
"resource": {
"token": "key",
"base_url": format!("http://127.0.0.1:{stub_port}/accounts/acc/ai/run/@cf/cloudflare")
},
"model": "clef-flash"
}),
json!({
"kind": "openai",
"resource": {"api_key": "key", "base_url": format!("http://127.0.0.1:{stub_port}/openai/v1")},
"model": "gpt-6-luna"
}),
] {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [
{"id": "d", "value": {
"type": "aidecision",
"input_transforms": {
"provider": {"type": "static", "value": provider},
"state": {"type": "static", "value": "I was charged twice"},
"questions": {"type": "static", "value": {"intent": {
"type": "choice",
"instructions": "What does the customer want?",
"criteria": {"refund": "Money back", "bug": "Something is broken"}
}}}
}
}},
{"id": "route", "value": {
"type": "branchone",
"branches": [
{
"expr": "results.d.output.intent.choice === 'bug'",
"modules": [{"id": "b", "value": {"type": "identity"}}]
},
{
"expr": "results.d.output.intent.choice === 'refund'",
"modules": [{"id": "r", "value": {
"type": "rawscript",
"language": "deno",
"content": "export function main(choice){ return 'handled ' + choice }",
"input_transforms": {"choice": {
"type": "javascript",
"expr": "results.d.output.intent.choice"
}}
}}]
}
],
"default": []
}}
]
}))?;
let job = run_job_in_new_worker_until_complete(
&db,
false,
JobPayload::RawFlow { value: flow, path: None, restarted_from: None },
port,
)
.await;
assert_eq!(
job.json_result(),
Some(json!("handled refund")),
"{provider}"
);
}
Ok(())
}
backend/tests/fixtures/jobs_read_auth.sql
Vendored
+7
View File
@@ -29,6 +29,13 @@ INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, sc
ARRAY['jobs:run:flows:f/shared/flow1']
);
-- A path-scoped read token for test-user-2: it reads the runs of `f/shared/flow1` only.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
encode(sha256('READ_SCOPED_TOKEN'::bytea), 'hex'), 'READ_SCOP', 'READ_SCOPED_TOKEN',
'test2@windmill.dev', 'flow history token', false,
ARRAY['jobs:read:f/shared/flow1']
);
-- Same, scoped to a script. The two jobs below both run through a `singlestepflow`
-- wrapper (native retry / scheduled runs produce these) — one wrapping a script, one
-- wrapping a flow — so the confinement has to project each onto the runnable it wraps
backend/tests/job_token_scopes.rs
+59
View File
@@ -17,6 +17,15 @@ async fn insert_job(
id: &str,
parent: Option<&str>,
scopes: &[&str],
) -> anyhow::Result<()> {
insert_job_with(db, id, parent, Some(scopes)).await
}
async fn insert_job_with(
db: &Pool<Postgres>,
id: &str,
parent: Option<&str>,
scopes: Option<&[&str]>,
) -> anyhow::Result<()> {
let id = Uuid::parse_str(id)?;
sqlx::query(
@@ -353,3 +362,53 @@ async fn test_deploy_without_the_field_keeps_the_restriction(
assert_eq!(stored("u/test-user/renamed").await?, None);
Ok(())
}
/// The OIDC `job_token_scopes` claim states the scopes of the token that asked for it.
#[cfg(all(feature = "enterprise", feature = "private", feature = "openidconnect"))]
#[sqlx::test(fixtures("base"))]
async fn test_oidc_claim_carries_the_presented_scopes(db: Pool<Postgres>) -> anyhow::Result<()> {
use base64::Engine;
initialize_tracing().await;
const OPEN_JOB: &str = "b0000000-0000-0000-0000-000000000005";
insert_script(&db, "u/test-user-3/agent", 535353, None).await?;
insert_job(&db, OIDC_JOB, None, &["oidc:write"]).await?;
insert_job_with(&db, OPEN_JOB, None, None).await?;
sqlx::query("UPDATE v2_job SET runnable_id = 535353 WHERE id = ANY($1)")
.bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?])
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, running, scheduled_for, tag)
SELECT id, workspace_id, true, now(), tag FROM v2_job WHERE id = ANY($1)",
)
.bind([Uuid::parse_str(OIDC_JOB)?, Uuid::parse_str(OPEN_JOB)?])
.execute(&db)
.await?;
let server = ApiServer::start(db.clone()).await?;
set_jwt_secret().await;
// The issuer of the minted token.
windmill_common::BASE_URL.store(std::sync::Arc::new("http://localhost".to_string()));
let base = format!(
"http://localhost:{}/api/w/test-workspace",
server.addr.port()
);
let client = reqwest::Client::new();
for (job, expected) in [(OIDC_JOB, json!(["oidc:write"])), (OPEN_JOB, json!(null))] {
let resp = client
.post(format!("{base}/oidc/token/test-audience"))
.bearer_auth(job_token(&db, job).await?)
.send()
.await?;
let status = resp.status();
let id_token = resp.text().await?;
assert_eq!(status, StatusCode::OK, "{id_token}");
let payload = id_token.split('.').nth(1).expect("a JWT payload");
let claims: serde_json::Value = serde_json::from_slice(
&base64::engine::general_purpose::URL_SAFE_NO_PAD.decode(payload)?,
)?;
assert_eq!(claims["job_token_scopes"], expected, "{job}");
}
Ok(())
}
backend/tests/jobs_read_auth.rs
+50
View File
@@ -564,6 +564,56 @@ async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Resul
"run-scoped token must not enumerate jobs (got {status}): {body}"
);
// ---- PATH-SCOPED READ TOKEN: READ_SCOPED_TOKEN is test-user-2's
// `jobs:read:f/shared/flow1`. It reads that flow's runs and the steps beneath
// them, and nothing else its owner created.
for (path, expected) in [
(
format!("completed/get_result/{FLOW_JOB}"),
r#""flow": "done""#,
),
(
format!("completed/get_result/{STEP_JOB}"),
"STEP_RESULT_INHERITED",
),
] {
let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await;
assert!(
status.is_success() && body.contains(expected),
"read-scoped token must read a run of its flow ({path}, got {status}): {body}"
);
}
for path in [
format!("completed/get_result/{VICTIM}"),
format!("get/{VICTIM}"),
format!("flow/approval_info/{VICTIM}"),
] {
let (status, body) = get(&base, &path, Some("READ_SCOPED_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"read-scoped token must not read a job outside its paths ({path}, got {status}): {body}"
);
}
// The lists answer with the top-level runs at its paths only, where the same user's
// unscoped token sees the other runs too.
for list in ["list", "completed/list"] {
let (status, body) = get(&authed_base, list, Some("READ_SCOPED_TOKEN")).await;
assert!(status.is_success(), "{list} (got {status}): {body}");
let jobs: Vec<serde_json::Value> = serde_json::from_str(&body)?;
let mut ids: Vec<&str> = jobs.iter().filter_map(|j| j["id"].as_str()).collect();
ids.sort();
assert_eq!(ids, [WRAPPED_FLOW_JOB, FLOW_JOB], "{list}: {body}");
}
let (_, body) = get(&authed_base, "list", Some("SECRET_TOKEN_2")).await;
assert!(body.contains(VICTIM), "unscoped list: {body}");
let (status, body) = get(&authed_base, "completed/count", Some("READ_SCOPED_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"read-scoped token must not count the workspace's jobs (got {status}): {body}"
);
// ---- APP EMBED TOKEN: cancellation confined to the app's own jobs. The token
// may cancel a job it launched (created_by == viewer), but `cancel_job_api`
// denies (NotFound) a job created by someone else, even one the (admin)
backend/tests/otel.rs
+6 -1
View File
@@ -168,6 +168,7 @@ async fn test_all_metrics_e2e() {
// ── Gauges ──────────────────────────────────────────────────────
otel_set_queue_count("python3", 42);
otel_set_queue_delay("python3", 12.5);
otel_set_queue_running_count("deno", 5);
otel_set_worker_busy("worker-test-1", 1);
otel_set_db_pool(5, 10, 20);
@@ -188,7 +189,7 @@ async fn test_all_metrics_e2e() {
let metrics = flush_and_get_metrics(&state);
let names = metric_names(&metrics);
// ── Verify all 20 metric names are present ──────────────────────
// ── Verify all 21 metric names are present ──────────────────────
let expected = [
"windmill.queue.push_count",
@@ -197,6 +198,7 @@ async fn test_all_metrics_e2e() {
"windmill.queue.zombie_restart_count",
"windmill.queue.zombie_delete_count",
"windmill.queue.count",
"windmill.queue.delay",
"windmill.queue.running_count",
"windmill.worker.execution_count",
"windmill.worker.execution_duration",
@@ -261,6 +263,9 @@ async fn test_all_metrics_e2e() {
.expect("queue.count data point with tag=python3 not found");
assert_eq!(dp.1, 42);
let m = find_metric(&metrics, "windmill.queue.delay").unwrap();
assert!((gauge_f64_value(m).unwrap() - 12.5).abs() < f64::EPSILON);
let m = find_metric(&metrics, "windmill.queue.running_count").unwrap();
let values = gauge_i64_values(m);
let dp = values
backend/tests/worker.rs
+123
View File
@@ -4698,6 +4698,129 @@ async fn test_run_wait_result_early_return_with_failure_module(
Ok(())
}
/// `recover: true` from the error handler of a step that fails inside a loop turns the flow
/// green, as it does for a top-level step, without changing which steps run: a loop that
/// stops at a failed iteration still stops there, one that skips failures still carries on.
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let failing_loop = |skip_failures: bool| {
json!({
"id": "loop",
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": [1, 2, 3] },
"skip_failures": skip_failures,
"modules": [{
"id": "a",
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"content": "export function main() { throw new Error('boom'); }",
},
}],
},
})
};
let run = |loop_step: serde_json::Value| {
let flow: FlowValue = serde_json::from_value(json!({
"modules": [loop_step, {
"id": "b",
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"content": "export function main() { return { ran_b: true } }",
},
}],
"failure_module": {
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"content": "export function main() { return { handled: true, recover: true } }",
},
},
}))
.unwrap();
RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
.run_until_complete(&db, false, port)
};
let handler_runs = || {
sqlx::query_scalar::<_, i64>(
"SELECT count(*) FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.kind = 'preview' AND c.result @> '{\"handled\": true}'::jsonb",
)
.fetch_one(&db)
};
let stopped = run(failing_loop(false)).await;
let stopped_handler_runs = handler_runs().await?;
let skipping = run(failing_loop(true)).await;
let skipping_handler_runs = handler_runs().await? - stopped_handler_runs;
// A parallel node counts its iterations' stored status: an iteration that ended on a
// recovered failure must still be stored as failed there.
let parallel = run(json!({
"id": "par",
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": [1, 2] },
"parallel": true,
"skip_failures": false,
"modules": [failing_loop(false)],
},
}))
.await;
// A recovery the inner loop absorbs, then an error raised starting the next step: that
// error is a new failure and must leave the flow failed.
#[cfg(feature = "quickjs")]
let chaining_error = run(json!({
"id": "outer",
"value": {
"type": "forloopflow",
"iterator": { "type": "static", "value": [1] },
"skip_failures": false,
"modules": [failing_loop(true), {
"id": "guarded",
"skip_if": { "expr": "missingFunction()" },
"value": { "input_transforms": {}, "type": "identity" },
}],
},
}))
.await;
server.close().await.unwrap();
#[cfg(feature = "quickjs")]
assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed");
assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success");
assert!(
stopped.json_result().unwrap().get("ran_b").is_none(),
"the step after the loop must not run, as without recovery"
);
assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration");
assert!(skipping.success);
assert_eq!(
skipping.json_result().unwrap(),
json!({ "ran_b": true }),
"a loop that skips failures runs every iteration and the following step"
);
assert_eq!(skipping_handler_runs, 3);
assert!(
!parallel.json_result().unwrap().to_string().contains("ran_b"),
"the parallel loop fails as without recovery, so the step after it must not run"
);
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_flow_lock_all(db: Pool<Postgres>) -> anyhow::Result<()> {
backend/windmill-ai/src/ai_providers.rs
+37
View File
@@ -39,6 +39,7 @@ lazy_static::lazy_static! {
pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1";
pub const DEEPSEEK_BASE_URL: &str = "https://api.deepseek.com/v1";
pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
pub const TYPESAFE_BASE_URL: &str = "https://api.typesafe.ai/v1";
/// Hosts that serve the OpenAI API with Azure conventions: Azure OpenAI
/// (`*.openai.azure.com`), AI Foundry (`*.services.ai.azure.com`,
@@ -122,9 +123,41 @@ pub enum AIProvider {
CustomAI,
#[serde(rename = "aws_bedrock")]
AWSBedrock,
/// TypeSafe's Jev decision model. It answers typed questions rather than chatting, so only
/// an AI decision runs it (`run_decision`), never the agent loop.
TypeSafe,
/// Cloudflare Workers AI, for its Jev-compatible decision models (Clef). Decisions only, as
/// for TypeSafe; its base URL comes from the resource's account id.
Cloudflare,
}
/// The Workers AI models URL of a Cloudflare account. The id goes into the path, so it is held to
/// the characters an account id has rather than trusted to stay inside its segment.
pub fn cloudflare_workers_ai_base_url(account_id: Option<&str>) -> Result<String> {
match account_id {
Some(id) if !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric()) => Ok(format!(
"https://api.cloudflare.com/client/v4/accounts/{id}/ai/run/@cf/cloudflare"
)),
Some(_) => Err(Error::BadRequest(
"The Cloudflare account_id must be letters and digits only".to_string(),
)),
None => Err(Error::BadRequest(
"A Cloudflare resource needs an account_id".to_string(),
)),
}
}
impl AIProvider {
/// A provider that answers decisions (typed questions) and serves no chat.
pub fn is_decision_provider(&self) -> bool {
matches!(self, AIProvider::TypeSafe | AIProvider::Cloudflare)
}
/// A provider an AI decision runs on: the decision providers, and OpenAI, whose Decisions
/// API sits next to its chat routes on the same resource.
pub fn runs_decisions(&self) -> bool {
self.is_decision_provider() || matches!(self, AIProvider::OpenAI)
}
/// Get the base URL for the AI provider
pub async fn get_base_url(&self, resource_base_url: Option<String>, db: &DB) -> Result<String> {
if let Some(base_url) = resource_base_url {
@@ -177,6 +210,10 @@ impl AIProvider {
AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()),
AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()),
AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()),
AIProvider::TypeSafe => Ok(TYPESAFE_BASE_URL.to_string()),
AIProvider::Cloudflare => Err(Error::BadRequest(
"A Cloudflare resource needs an account_id".to_string(),
)),
p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI | AIProvider::AzureFoundry) => {
Err(Error::BadRequest(format!(
"{:?} provider requires a base URL in the resource",
backend/windmill-ai/src/decision.rs
+556
View File
@@ -0,0 +1,556 @@
//! AI decision steps: one call to a decision endpoint, which answers typed questions (`choice`,
//! `score`, `noul`) about a state with calibrated probabilities instead of text. The questions and
//! answers are System One's (TypeSafe's Jev, Cloudflare's Jev-compatible Clef on Workers AI);
//! OpenAI's Decisions API is asked and read through a translation to that shape.
use std::time::Duration;
use serde::{Deserialize, Serialize};
use serde_json::{value::RawValue, Map, Value};
use windmill_common::error::{Error, Result};
use crate::{
ai_providers::AIProvider,
credentials::ProviderCredentials,
proxy::{common_outbound_headers, retain_effective_credentials},
types::{ProviderWithResource, TokenUsage},
utils::pinned_ai_client_for,
};
/// The inputs of an AI decision step or tool, as its `input_transforms` resolve them.
#[derive(Deserialize, Debug)]
pub struct AIDecisionArgs {
pub provider: ProviderWithResource,
#[serde(default)]
pub state: Option<Value>,
#[serde(default)]
pub questions: Option<Value>,
}
/// The alias TypeSafe keeps on its current model, sent when the step names none.
pub const TYPESAFE_DEFAULT_MODEL: &str = "jev-latest";
/// Cloudflare's larger decision model, run when the step names none.
pub const CLOUDFLARE_DEFAULT_MODEL: &str = "clef";
/// The model OpenAI's Decisions API serves, run when the step names none.
pub const OPENAI_DEFAULT_MODEL: &str = "gpt-6-luna";
#[derive(Serialize)]
struct SystemOneRequest<'a> {
model: &'a str,
state: &'a Value,
questions: &'a Map<String, Value>,
}
#[derive(Deserialize)]
struct SystemOneResponse {
#[serde(default)]
model: Option<String>,
answers: Box<RawValue>,
#[serde(default)]
usage: Option<SystemOneUsage>,
}
/// Workers AI wraps the System One response in its API envelope.
#[derive(Deserialize)]
struct CloudflareEnvelope {
result: SystemOneResponse,
}
#[derive(Deserialize)]
struct SystemOneUsage {
input_tokens: Option<i32>,
output_tokens: Option<i32>,
}
/// A decision's result. `output` holds the answers keyed by question name, where an agent's
/// holds its answer, so a later step reads both as `results.<step>.output`.
#[derive(Serialize)]
pub struct DecisionResult {
pub output: Box<RawValue>,
/// The version that answered, which `jev-latest` resolves to.
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<TokenUsage>,
}
/// The state and questions of a decision, refused here rather than by the endpoint when
/// either is missing or of a shape TypeSafe rejects: a flow that passes an empty expression result
/// should read what it forgot, without a request going out.
pub fn decision_inputs<'a>(
state: Option<&'a Value>,
questions: Option<&'a Value>,
) -> Result<(&'a Value, &'a Map<String, Value>)> {
let state = match state {
None | Some(Value::Null) => {
return Err(Error::BadRequest(
"'state' must be provided for an AI decision".to_string(),
))
}
Some(Value::String(s)) if s.trim().is_empty() => {
return Err(Error::BadRequest(
"'state' must be provided for an AI decision".to_string(),
))
}
Some(state @ (Value::String(_) | Value::Object(_) | Value::Array(_))) => state,
Some(_) => {
return Err(Error::BadRequest(
"'state' must be a text, an object or an array for an AI decision".to_string(),
))
}
};
let questions = match questions {
Some(Value::Object(questions)) if !questions.is_empty() => questions,
_ => {
return Err(Error::BadRequest(
"'questions' must be an object naming at least one question for an AI decision"
.to_string(),
))
}
};
Ok((state, questions))
}
/// Posts a decision request with the resource's bearer key and returns the response body.
async fn post_decision<T: Serialize>(
credentials: &ProviderCredentials,
endpoint: &str,
vendor: &str,
payload: &T,
timeout: Duration,
) -> Result<String> {
let auth_headers = retain_effective_credentials(
credentials,
vec![(
"Authorization",
format!(
"Bearer {}",
credentials.api_key.as_deref().unwrap_or_default()
),
)],
);
// The endpoint derives from the resource's base URL, so the connect is pinned to the
// address the SSRF check validated.
let client = pinned_ai_client_for(&credentials.base_url).await?;
let mut request = client.post(endpoint).timeout(timeout);
for (name, value) in auth_headers {
request = request.header(name, value);
}
for (name, value) in common_outbound_headers(credentials) {
request = request.header(name, value);
}
let response = request
.json(payload)
.send()
.await
.map_err(|e| Error::ExecutionErr(format!("Failed to call {vendor}: {e}")))?;
let status = response.status();
if !status.is_success() {
let body = response
.text()
.await
.unwrap_or_else(|_| "<failed to read body>".to_string());
return Err(Error::ExecutionErr(format!(
"{vendor} error calling {endpoint}: {status} - {body}"
)));
}
response
.text()
.await
.map_err(|e| Error::ExecutionErr(format!("Failed to read the {vendor} response: {e}")))
}
/// Answers a decision on the endpoint its provider serves.
pub async fn run_decision(
credentials: &ProviderCredentials,
model: &str,
state: &Value,
questions: &Map<String, Value>,
timeout: Duration,
) -> Result<DecisionResult> {
if credentials.provider == AIProvider::OpenAI {
run_openai_decisions(credentials, model, state, questions, timeout).await
} else {
run_systemone(credentials, model, state, questions, timeout).await
}
}
#[derive(Serialize)]
struct OpenAIDecisionsRequest<'a> {
model: &'a str,
input: Value,
questions: Vec<Value>,
}
#[derive(Deserialize)]
struct OpenAIDecisionsResponse {
#[serde(default)]
model: Option<String>,
answers: Vec<Map<String, Value>>,
#[serde(default)]
usage: Option<SystemOneUsage>,
}
/// The state as a Decisions `input`. OpenAI reads a text or a list of user messages, so a list of
/// user messages (the only way to pass an image) goes as it is and any other object or array,
/// a chat history included, as its JSON text.
fn openai_input(state: &Value) -> Value {
match state {
Value::String(_) => state.clone(),
Value::Array(items) if !items.is_empty() && items.iter().all(is_openai_user_message) => {
state.clone()
}
_ => Value::String(state.to_string()),
}
}
/// Exactly a message the Decisions API reads: `role: "user"` with a text, or with parts that are
/// each an `input_text` or an `input_image`.
fn is_openai_user_message(message: &Value) -> bool {
let is_part = |part: &Value| match part.get("type").and_then(Value::as_str) {
Some("input_text") => part.get("text").is_some_and(Value::is_string),
Some("input_image") => part.get("image_url").is_some_and(Value::is_string),
_ => false,
};
message.get("role").and_then(Value::as_str) == Some("user")
&& match message.get("content") {
Some(Value::String(_)) => true,
Some(Value::Array(parts)) => !parts.is_empty() && parts.iter().all(is_part),
_ => false,
}
}
/// The questions as OpenAI's Decisions API takes them: a list naming each question, with a
/// `noul` asked as its `predicate` and each score level as a label.
fn openai_questions(questions: &Map<String, Value>) -> Result<Vec<Value>> {
let text = |v: &Value| match v {
Value::String(s) => s.clone(),
v => v.to_string(),
};
// A structured description goes as its JSON text, as structured instructions do.
let stated = |description: Option<&Value>| match description {
None | Some(Value::Null) => None,
Some(Value::String(d)) if d.trim().is_empty() => None,
Some(d) => Some(text(d)),
};
questions
.iter()
.map(|(name, question)| {
let criteria = question.get("criteria");
let mut instructions = question.get("instructions").map(text).unwrap_or_default();
let mut asked = Map::new();
let kind = match question.get("type").and_then(Value::as_str) {
Some("choice") => {
let choices = criteria
.and_then(Value::as_object)
.filter(|c| !c.is_empty())
.ok_or_else(|| {
Error::BadRequest(format!(
"The choice question '{name}' needs its options as 'criteria'"
))
})?
.iter()
.map(|(value, d)| match stated(Some(d)) {
Some(d) => serde_json::json!({"value": value, "description": d}),
None => serde_json::json!({"value": value}),
})
.collect();
asked.insert("choices".to_string(), Value::Array(choices));
"choice"
}
Some("score") => {
let levels = criteria
.and_then(Value::as_array)
.filter(|c| !c.is_empty())
.ok_or_else(|| {
Error::BadRequest(format!(
"The score question '{name}' needs its levels as 'criteria'"
))
})?
.iter()
.map(|level| serde_json::json!({"label": text(level)}))
.collect();
asked.insert("levels".to_string(), Value::Array(levels));
"score"
}
Some("noul") => {
for (key, meaning) in [("true", "Yes"), ("false", "No")] {
if let Some(d) = stated(criteria.and_then(|c| c.get(key))) {
instructions.push_str(&format!("\n{meaning}: {d}"));
}
}
"predicate"
}
other => {
return Err(Error::BadRequest(format!(
"The question '{name}' has type {}, which is not 'choice', 'score' or 'noul'",
other.map_or("none".to_string(), |t| format!("'{t}'"))
)))
}
};
asked.insert("type".to_string(), kind.into());
asked.insert("name".to_string(), name.as_str().into());
asked.insert("instructions".to_string(), instructions.into());
Ok(Value::Object(asked))
})
.collect()
}
/// OpenAI's answers keyed by question name, as every decision's are. A predicate's probability is
/// also given as `noul`, so a flow reads a yes/no answer the same way whichever provider gave it.
/// A refused question fails the decision: left in the output it has no `choice` or `noul`, so a
/// branch on it would quietly take its "no" or default arm.
fn openai_answers(answers: Vec<Map<String, Value>>) -> Result<Map<String, Value>> {
let mut by_name = Map::new();
for mut answer in answers {
let Some(Value::String(name)) = answer.remove("name") else {
continue;
};
match answer.get("type").and_then(Value::as_str) {
Some("refusal") => {
return Err(Error::ExecutionErr(format!(
"OpenAI refused to answer the question '{name}': {}",
Value::Object(answer)
)))
}
Some("predicate") => {
if let Some(probability) = answer.get("probability").cloned() {
answer.insert("noul".to_string(), probability);
}
}
_ => {}
}
by_name.insert(name, Value::Object(answer));
}
Ok(by_name)
}
async fn run_openai_decisions(
credentials: &ProviderCredentials,
model: &str,
state: &Value,
questions: &Map<String, Value>,
timeout: Duration,
) -> Result<DecisionResult> {
let model = match model.trim() {
"" => OPENAI_DEFAULT_MODEL,
model => model,
};
let endpoint = format!("{}/decisions", credentials.base_url.trim_end_matches('/'));
let body = post_decision(
credentials,
&endpoint,
"OpenAI",
&OpenAIDecisionsRequest {
model,
input: openai_input(state),
questions: openai_questions(questions)?,
},
timeout,
)
.await?;
let parsed = serde_json::from_str::<OpenAIDecisionsResponse>(&body)
.map_err(|e| Error::ExecutionErr(format!("Unexpected OpenAI response ({e}): {body}")))?;
Ok(DecisionResult {
output: serde_json::value::to_raw_value(&openai_answers(parsed.answers)?)
.map_err(|e| Error::internal_err(format!("serializing decision answers: {e}")))?,
model: parsed.model,
usage: parsed
.usage
.map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens))
.filter(|u| !u.is_empty()),
})
}
async fn run_systemone(
credentials: &ProviderCredentials,
model: &str,
state: &Value,
questions: &Map<String, Value>,
timeout: Duration,
) -> Result<DecisionResult> {
let cloudflare = credentials.provider == AIProvider::Cloudflare;
let model = match model.trim() {
"" if cloudflare => CLOUDFLARE_DEFAULT_MODEL,
"" => TYPESAFE_DEFAULT_MODEL,
model => model,
};
let base = credentials.base_url.trim_end_matches('/');
// Workers AI serves each model at its own URL; the model goes into the path, so it is held to
// the characters a model id has.
let endpoint = if cloudflare {
if !model
.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))
{
return Err(Error::BadRequest(format!(
"Invalid Cloudflare decision model: {model}"
)));
}
format!("{base}/{model}")
} else {
format!("{base}/systemone")
};
let vendor = if cloudflare { "Cloudflare" } else { "TypeSafe" };
let body = post_decision(
credentials,
&endpoint,
vendor,
&SystemOneRequest { model, state, questions },
timeout,
)
.await?;
// Tried in turn rather than as an untagged enum, which cannot hold the raw `answers`.
let parsed = match serde_json::from_str::<SystemOneResponse>(&body) {
Ok(parsed) => parsed,
Err(e) => match serde_json::from_str::<CloudflareEnvelope>(&body) {
Ok(envelope) => envelope.result,
Err(_) => {
return Err(Error::ExecutionErr(format!(
"Unexpected {vendor} response ({e}): {body}"
)))
}
},
};
Ok(DecisionResult {
output: parsed.answers,
model: parsed.model,
// Dropped when empty, as an agent's is, so `usage` is either counted or absent.
usage: parsed
.usage
.map(|u| TokenUsage::from_input_output(u.input_tokens, u.output_tokens))
.filter(|u| !u.is_empty()),
})
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
/// The account id goes into the URL path, so anything that could leave its segment is refused.
#[test]
fn cloudflare_account_id_stays_in_its_segment() {
use crate::ai_providers::cloudflare_workers_ai_base_url;
assert_eq!(
cloudflare_workers_ai_base_url(Some("abc123")).unwrap(),
"https://api.cloudflare.com/client/v4/accounts/abc123/ai/run/@cf/cloudflare"
);
for id in [None, Some("a/b"), Some("../x"), Some("a?b"), Some("")] {
assert!(cloudflare_workers_ai_base_url(id).is_err(), "{id:?}");
}
}
/// A `noul` goes out as a predicate with its yes/no meanings in the instructions, and comes
/// back readable as `noul`; choices and levels go out as lists, answers come back by name.
#[test]
fn openai_questions_and_answers_translate() {
let questions = json!({
"intent": {"type": "choice", "instructions": "Why?", "criteria": {"refund": {"when": "paid"}, "bug": ""}},
"urgency": {"type": "score", "instructions": "How urgent?", "criteria": ["Can wait", "Now"]},
"angry": {"type": "noul", "instructions": "Angry?", "criteria": {"true": "Insults"}}
});
let asked = Value::Array(openai_questions(questions.as_object().unwrap()).unwrap());
assert_eq!(
asked,
json!([
{"choices": [
{"value": "refund", "description": "{\"when\":\"paid\"}"},
{"value": "bug"}
], "type": "choice", "name": "intent", "instructions": "Why?"},
{"levels": [
{"label": "Can wait"},
{"label": "Now"}
], "type": "score", "name": "urgency", "instructions": "How urgent?"},
{"type": "predicate", "name": "angry", "instructions": "Angry?\nYes: Insults"}
])
);
for bad in [
json!({"q": {"type": "choice", "instructions": "?"}}),
json!({"q": {"type": "score", "instructions": "?", "criteria": []}}),
json!({"q": {"type": "predicate", "instructions": "?"}}),
] {
assert!(openai_questions(bad.as_object().unwrap()).is_err(), "{bad}");
}
let answers = serde_json::from_value(json!([
{"type": "predicate", "name": "angry", "probability": 0.92},
{"type": "choice", "name": "intent", "choice": "refund", "confidence": 0.9}
]))
.unwrap();
assert_eq!(
Value::Object(openai_answers(answers).unwrap()),
json!({
"angry": {"type": "predicate", "probability": 0.92, "noul": 0.92},
"intent": {"type": "choice", "choice": "refund", "confidence": 0.9}
})
);
let messages = json!([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]);
assert_eq!(openai_input(&messages), messages);
assert_eq!(openai_input(&json!({"m": "hi"})), json!("{\"m\":\"hi\"}"));
assert_eq!(
openai_input(&json!([{"role": "admin"}])),
json!("[{\"role\":\"admin\"}]")
);
for records in [
json!([{"role": "user", "content": {"ticket": "charged twice"}}]),
json!([{"role": "user", "content": null}]),
json!([{"role": "user", "content": ["a"]}]),
json!([{"role": "user", "content": [{"type": "input_text", "text": {"t": 1}}]}]),
json!([{"role": "user", "content": [{"type": "input_image"}]}]),
] {
assert_eq!(openai_input(&records), json!(records.to_string()));
}
assert_eq!(
openai_input(&json!([{"role": "user", "content": "Hi"}])),
json!([{"role": "user", "content": "Hi"}])
);
// A chat history is evaluated as text: OpenAI takes user messages only.
let history =
json!([{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Yo"}]);
assert_eq!(openai_input(&history), json!(history.to_string()));
let refused =
serde_json::from_value(json!([{"type": "refusal", "name": "angry"}])).unwrap();
assert!(openai_answers(refused).is_err());
}
/// Refused are the values a flow produces when it forgot the input (no key, `null`, a blank
/// string from an empty expression) and the scalars TypeSafe rejects. A text, an object or an
/// array is a state, whatever it holds.
#[test]
fn decision_inputs_refuse_what_typesafe_would() {
let questions = json!({"urgent": {"type": "noul", "instructions": "Is it urgent?"}});
for state in [
None,
Some(json!(null)),
Some(json!(" ")),
Some(json!(false)),
Some(json!(0)),
] {
let err = decision_inputs(state.as_ref(), Some(&questions)).unwrap_err();
assert!(err.to_string().contains("'state'"), "{err}");
}
for state in [json!("0"), json!({}), json!([]), json!({"m": "hi"})] {
assert!(
decision_inputs(Some(&state), Some(&questions)).is_ok(),
"{state}"
);
}
for questions in [
None,
Some(json!({})),
Some(json!([])),
Some(json!("urgent")),
] {
let err = decision_inputs(Some(&json!("hi")), questions.as_ref()).unwrap_err();
assert!(err.to_string().contains("'questions'"), "{err}");
}
}
}
backend/windmill-ai/src/image_handler.rs
+9 -1
View File
@@ -47,6 +47,9 @@ pub async fn upload_image_to_s3(
})
}
// Bounds each attachment's download before base64 expansion; provider limits may be lower.
const MAX_S3_MEDIA_BYTES: usize = 30 * 1024 * 1024;
/// Download an S3 image and convert it to a base64 data URL.
///
/// The caller must provide an AuthedClient authorized for `workspace_id`.
@@ -57,7 +60,12 @@ pub async fn download_and_encode_s3_image(
) -> Result<(String, String), Error> {
// Download the image from S3
let image_bytes = client
.download_s3_file(workspace_id, &image.s3, image.storage.clone())
.download_s3_file(
workspace_id,
&image.s3,
image.storage.clone(),
Some(MAX_S3_MEDIA_BYTES),
)
.await
.map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?;
backend/windmill-ai/src/lib.rs
+1
View File
@@ -5,6 +5,7 @@ pub mod ai_google;
pub mod ai_providers;
pub mod ai_types;
pub mod credentials;
pub mod decision;
pub mod image_handler;
pub mod model_context;
pub mod providers;
backend/windmill-ai/src/providers/mod.rs
+6
View File
@@ -204,6 +204,12 @@ pub fn create_query_builder(
AIProvider::AzureFoundry if AIProvider::is_anthropic_model(model) => Box::new(
AnthropicQueryBuilder::new(AIProvider::AzureFoundry, AIPlatform::Standard),
),
// Only for forwarding through the API proxy, which is bearer auth on `<base>/<path>` as
// TypeSafe and Cloudflare expect. Neither serves a chat route for its decision models, and
// `run_agent` refuses both before a chat request could be built.
AIProvider::TypeSafe | AIProvider::Cloudflare => {
Box::new(OtherQueryBuilder::new(credentials.provider.clone()))
}
_ => Box::new(OtherQueryBuilder::new(credentials.provider.clone())),
}
}
backend/windmill-ai/src/proxy.rs
+5 -1
View File
@@ -72,7 +72,9 @@ pub fn proxy_execution_mode(provider: &AIProvider) -> ProxyExecutionMode {
| AIProvider::Groq
| AIProvider::OpenRouter
| AIProvider::TogetherAI
| AIProvider::CustomAI => ProxyExecutionMode::HttpForward,
| AIProvider::CustomAI
| AIProvider::TypeSafe
| AIProvider::Cloudflare => ProxyExecutionMode::HttpForward,
AIProvider::GoogleAI => ProxyExecutionMode::NativeGoogleAi,
AIProvider::AWSBedrock => ProxyExecutionMode::NativeAwsBedrock,
}
@@ -444,6 +446,8 @@ mod tests {
(AIProvider::OpenRouter, ProxyExecutionMode::HttpForward),
(AIProvider::TogetherAI, ProxyExecutionMode::HttpForward),
(AIProvider::CustomAI, ProxyExecutionMode::HttpForward),
(AIProvider::TypeSafe, ProxyExecutionMode::HttpForward),
(AIProvider::Cloudflare, ProxyExecutionMode::HttpForward),
(AIProvider::GoogleAI, ProxyExecutionMode::NativeGoogleAi),
(AIProvider::AWSBedrock, ProxyExecutionMode::NativeAwsBedrock),
];
backend/windmill-ai/src/query_builder.rs
+36 -1
View File
@@ -1,5 +1,7 @@
use std::borrow::Cow;
use async_trait::async_trait;
use windmill_common::{client::AuthedClient, error::Error};
use windmill_common::{client::AuthedClient, error::Error, utils::strip_json_nul};
use windmill_types::s3::S3Object;
use crate::ai_types::OpenAIToolCall;
@@ -92,6 +94,39 @@ pub enum ParsedResponse {
},
}
impl ParsedResponse {
/// Drops every U+0000 the model emitted. Postgres stores neither a NUL in `text` nor its
/// `\u0000` escape in `jsonb`, and this output is written to both: tool job args, the flow
/// status, agent memory, conversation rows.
pub fn without_nul(mut self) -> Self {
fn strip(s: &mut String) {
if s.contains('\0') {
s.retain(|c| c != '\0');
}
}
if let ParsedResponse::Text { content, reasoning, tool_calls, annotations, .. } = &mut self
{
for text in [content, reasoning].into_iter().flatten() {
strip(text);
}
for annotation in annotations {
strip(&mut annotation.url);
annotation.title.as_mut().into_iter().for_each(strip);
}
for tool_call in tool_calls {
strip(&mut tool_call.id);
strip(&mut tool_call.function.name);
// Arguments are serialized JSON, where a NUL is spelled as its escape.
strip(&mut tool_call.function.arguments);
if let Cow::Owned(stripped) = strip_json_nul(&tool_call.function.arguments) {
tool_call.function.arguments = stripped;
}
}
}
self
}
}
/// Trait for streaming AI events to a sink (e.g., database persistence).
/// Implemented by the worker's StreamEventProcessor.
#[async_trait]
backend/windmill-ai/src/types.rs
+19 -1
View File
@@ -269,6 +269,12 @@ pub struct ProviderResource {
/// credentials exchange this crate does not perform.
#[serde(default, deserialize_with = "empty_string_as_none")]
pub token_url: Option<String>,
/// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`.
#[serde(default, deserialize_with = "empty_string_as_none")]
pub token: Option<String>,
/// The account of a `cloudflare` resource, which its Workers AI URL is built from.
#[serde(default, deserialize_with = "empty_string_as_none")]
pub account_id: Option<String>,
}
#[derive(Deserialize, Debug)]
@@ -301,6 +307,11 @@ impl ProviderWithResource {
}
pub async fn get_base_url(&self, db: &DB) -> Result<String, Error> {
if self.kind == AIProvider::Cloudflare && self.resource.base_url.is_none() {
return crate::ai_providers::cloudflare_workers_ai_base_url(
self.resource.account_id.as_deref(),
);
}
self.kind
.get_base_url(self.resource.base_url.clone(), db)
.await
@@ -320,7 +331,14 @@ impl ProviderWithResource {
Ok(ProviderCredentials {
provider: self.kind.clone(),
base_url,
api_key: self.resource.api_key.clone(),
api_key: match self.kind {
AIProvider::Cloudflare => self
.resource
.api_key
.clone()
.or_else(|| self.resource.token.clone()),
_ => self.resource.api_key.clone(),
},
access_token: None,
organization_id: None,
user: None,
backend/windmill-api-auth/src/auth.rs
+1 -1
View File
@@ -107,7 +107,7 @@ pub struct ExpiringAuthCache {
/// Whether `target` belongs to the flow run of the job `job_id`: the job itself, one of its
/// ancestors, or a job whose parent or root is one of them (a sibling step, a loop or branch
/// iteration, any job pushed as a child of one of them). A lookup that fails reads as no.
async fn job_in_same_flow_run(db: &DB, job_id: uuid::Uuid, target: uuid::Uuid) -> bool {
pub async fn job_in_same_flow_run(db: &DB, job_id: uuid::Uuid, target: uuid::Uuid) -> bool {
if job_id == target {
return true;
}
backend/windmill-api-auth/src/scopes.rs
+287 -53
View File
@@ -302,7 +302,7 @@ fn extract_domain_from_route(
/// Every entry is keyed by a job id and confines an authenticated caller to its own
/// runnable — through `require_job_read_access`, through its own `jobs:run:flows:<path>`
/// check, or, where an approval token or resume secret bypasses that gate, through a
/// direct `require_job_within_run_scope`. The one exception is
/// direct `require_job_within_read_scope`. The one exception is
/// `jobs_u/get_root_job_id/`, which has no check at all but discloses only flow lineage,
/// to anyone, authenticated or not. Workspace-wide enumeration (`jobs/list`, counts,
/// exports) and credential minting (`job_view_token`) are deliberately absent — those are
@@ -519,23 +519,163 @@ fn resource_metadata_route_allowed(suffix: &str) -> bool {
|| suffix.starts_with("resources/type/")
}
/// The `jobs:run` scopes a token's job reads are confined to, or `None` when they are
/// not confined to particular runnables.
/// The by-id and list routes a path-scoped `jobs:read:<paths>` reaches, as workspaced
/// route suffixes. Every other read in the domain is refused to it: counts, exports,
/// uuid listings, queue positions and dispatch edges aggregate over the workspace, and
/// the view-token and resume-url routes mint a credential (`approval_info` withholds
/// its view token from a confined caller for the same reason).
///
/// Each entry confines the caller in its handler, through `require_job_read_access` or
/// a direct `require_job_within_read_scope`. `jobs/result_by_id/` is absent because it
/// resolves a node up through the enclosing flows, past the job the check admitted.
/// A route added here without that check serves a path-scoped token every job.
const READ_PATH_SCOPED_BY_ID_PATHS: [&'static str; 25] = [
"jobs_u/get/",
"jobs_u/get_logs/",
"jobs_u/get_completed_logs_tail/",
"jobs_u/get_flow_all_logs/",
"jobs_u/get_flow_all_logs_structured/",
"jobs_u/get_flow_all_results/",
"jobs_u/get_args/",
"jobs_u/get_flow_debug_info/",
"jobs_u/completed/get/",
"jobs_u/completed/get_result/",
"jobs_u/completed/get_result_maybe/",
"jobs_u/completed/get_timing/",
"jobs_u/dispatch_events/",
"jobs_u/getupdate/",
"jobs_u/getupdate_sse/",
"jobs_u/get_log_file/",
"jobs_u/get_flow/",
"jobs_u/flow/approval_info/",
"jobs/run_progress/",
"jobs/flow/user_states/",
"jobs/completed/get/",
"jobs/completed/get_result/",
"jobs/completed/get_result_maybe/",
"jobs/completed/get_timing/",
"jobs/get_otel_traces/",
];
/// Matched whole: `jobs/list` is also a prefix of the uuid listings. Their handlers
/// filter in SQL to [`job_list_path_filter`].
const READ_PATH_SCOPED_LIST_PATHS: [&'static str; 3] =
["jobs/list", "jobs/completed/list", "jobs/queue/list"];
fn is_read_path_scoped_route(suffix: &str) -> bool {
READ_PATH_SCOPED_LIST_PATHS.contains(&suffix)
|| READ_PATH_SCOPED_BY_ID_PATHS
.iter()
.any(|p| suffix.starts_with(p))
}
/// The paths of a `jobs:read:<paths>` scope, or `None` when it spans every job: no
/// path part, or `*` among them.
fn job_read_scope_paths(scope: &ScopeDefinition) -> Option<&[String]> {
scope
.resource
.as_deref()
.filter(|paths| !paths.iter().any(|p| p == "*"))
}
/// The runnables a token's job reads are confined to: those its `jobs:run` scopes may
/// start, and those its `jobs:read:<paths>` scopes name.
pub struct JobReadConfinement {
run: Vec<ScopeDefinition>,
read_paths: crate::ScopePathFilter,
}
impl JobReadConfinement {
/// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`)
/// is inside the confinement. A read path admits every kind; a run scope only its own.
pub fn admits(&self, kind: &str, runnable_path: &str) -> bool {
if self.read_paths.allows(runnable_path) {
return true;
}
let required = ScopeDefinition::new(
ScopeDomain::Jobs.as_str(),
ScopeAction::Run.as_str(),
Some(kind),
Some(vec![runnable_path.to_string()]),
);
self.run.iter().any(|scope| scope.includes(&required))
}
/// Whether a `jobs:read:<paths>` scope is part of the confinement.
pub fn has_read_paths(&self) -> bool {
matches!(&self.read_paths, crate::ScopePathFilter::Restricted { exact, prefix }
if !exact.is_empty() || !prefix.is_empty())
}
}
fn push_read_paths(paths: &[String], exact: &mut Vec<String>, prefix: &mut Vec<String>) {
for path in paths {
match path.strip_suffix("/*") {
Some(p) => prefix.push(p.to_string()),
None => exact.push(path.clone()),
}
}
}
/// The paths a token's job lists are filtered to. Decided by its read grants alone,
/// unlike [`job_read_confinement`]: a `jobs:run` scope grants no enumeration, so even a
/// bare one beside a `jobs:read:<paths>` leaves the lists on those paths. Unrestricted
/// for an unscoped token, a `jobs:read` naming no path (or `*`), and `jobs:write`.
pub fn job_list_path_filter(scopes: Option<&[String]>) -> crate::ScopePathFilter {
let mut exact = Vec::new();
let mut prefix = Vec::new();
let mut scoped = false;
for scope in scopes
.unwrap_or_default()
.iter()
.filter(|s| !s.starts_with("if_jobs:filter_tags:"))
{
scoped = true;
let Ok(scope) = ScopeDefinition::from_scope_string(scope) else {
continue;
};
if ScopeDomain::from_str(&scope.domain) != Some(ScopeDomain::Jobs) {
continue;
}
match ScopeAction::from_str(&scope.action) {
Some(ScopeAction::Read) => match job_read_scope_paths(&scope) {
Some(paths) => push_read_paths(paths, &mut exact, &mut prefix),
None => return crate::ScopePathFilter::AllowAll,
},
Some(ScopeAction::Write) => return crate::ScopePathFilter::AllowAll,
_ => continue,
}
}
if scoped {
crate::ScopePathFilter::Restricted { exact, prefix }
} else {
crate::ScopePathFilter::AllowAll
}
}
/// What a token's job reads are confined to, or `None` when they are not confined to
/// particular runnables.
///
/// A run scope is what the trigger UI mints per script or flow and hands to a webhook
/// caller / CI job: it may start the runnables it names and follow those runs, so its
/// by-id job reads must stay within what it can start (enforced by
/// `require_job_read_access`). Both the path (`jobs:run:flows:f/team/etl`) and the
/// kind-only (`jobs:run:scripts`, which legacy `jobs:runscript` tokens carry) forms
/// confine, since `ScopeDefinition::includes` already matches a candidate
/// `jobs:run:<kind>:<path>` against either.
/// by-id job reads must stay within what it can start. Both the path
/// (`jobs:run:flows:f/team/etl`) and the kind-only (`jobs:run:scripts`, which legacy
/// `jobs:runscript` tokens carry) forms confine, since `ScopeDefinition::includes`
/// already matches a candidate `jobs:run:<kind>:<path>` against either.
///
/// A `jobs:read:<paths>` scope reads the runs of the runnables at those paths and
/// nothing else. The two kinds of grant add up: a job either admits is readable.
///
/// Returns `None` — unconfined — when the token is effectively unscoped, or carries a
/// jobs scope that grants job reads in its own right: `jobs:read`/`jobs:write`, or a
/// bare `jobs:run` (it can start anything, so confining its reads to "what it may run"
/// would restrict nothing).
pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option<Vec<ScopeDefinition>> {
let mut confinement = Vec::new();
/// jobs scope that grants every job read in its own right: a `jobs:read` naming no path
/// (or `*`), a bare `jobs:run` (it can start anything, so confining its reads to "what
/// it may run" would restrict nothing), or `jobs:write`, whose paths are ignored on
/// reads as they are on every job write.
pub fn job_read_confinement(scopes: Option<&[String]>) -> Option<JobReadConfinement> {
let mut run = Vec::new();
let mut exact = Vec::new();
let mut prefix = Vec::new();
let mut confined = false;
for scope in scopes?
.iter()
.filter(|s| !s.starts_with("if_jobs:filter_tags:"))
@@ -548,7 +688,15 @@ pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option<Vec<ScopeDe
}
match ScopeAction::from_str(&scope.action) {
Some(ScopeAction::Run) if scope.kind.is_some() || scope.resource.is_some() => {
confinement.push(scope)
confined = true;
run.push(scope)
}
Some(ScopeAction::Read) => {
let Some(paths) = job_read_scope_paths(&scope) else {
return None;
};
confined = true;
push_read_paths(paths, &mut exact, &mut prefix);
}
// Grants no reads (see `scope_grants_access`), so it neither confines nor
// frees them.
@@ -557,23 +705,10 @@ pub fn job_read_run_confinement(scopes: Option<&[String]>) -> Option<Vec<ScopeDe
None => continue,
}
}
(!confinement.is_empty()).then_some(confinement)
}
/// Whether a job that ran `runnable_path` as `kind` (`scripts` or `flows`) is inside a
/// [`job_read_run_confinement`] set.
pub fn run_confinement_admits(
confinement: &[ScopeDefinition],
kind: &str,
runnable_path: &str,
) -> bool {
let required = ScopeDefinition::new(
ScopeDomain::Jobs.as_str(),
ScopeAction::Run.as_str(),
Some(kind),
Some(vec![runnable_path.to_string()]),
);
confinement.iter().any(|scope| scope.includes(&required))
confined.then_some(JobReadConfinement {
run,
read_paths: crate::ScopePathFilter::Restricted { exact, prefix },
})
}
/// The `jobs:cancel:<paths>` scopes a token's cancels are confined to, or `None` when
@@ -683,6 +818,17 @@ fn scope_grants_access(
}));
}
// A path-scoped `jobs:read:<paths>` is the one job scope this layer does not treat
// as resource-blind: it reaches only the routes whose handlers confine it to those
// paths, so a job read route is refused to it until it is listed there.
if scope_domain == ScopeDomain::Jobs
&& scope_action == ScopeAction::Read
&& required_action == ScopeAction::Read
&& job_read_scope_paths(scope).is_some()
{
return Ok(route_path.is_some_and(is_read_path_scoped_route));
}
if !scope_action.includes(&required_action) {
return Ok(false);
}
@@ -1137,28 +1283,16 @@ mod tests {
#[test]
fn run_scopes_confine_job_reads_by_kind_and_path() {
let confinement =
job_read_run_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap();
assert!(run_confinement_admits(&confinement, "flows", "f/team/etl"));
job_read_confinement(Some(&["jobs:run:flows:f/team/*".to_string()])).unwrap();
assert!(confinement.admits("flows", "f/team/etl"));
// Right path, wrong kind — a script named like the flow is not the flow.
assert!(!run_confinement_admits(
&confinement,
"scripts",
"f/team/etl"
));
assert!(!run_confinement_admits(
&confinement,
"flows",
"f/other/etl"
));
assert!(!confinement.admits("scripts", "f/team/etl"));
assert!(!confinement.admits("flows", "f/other/etl"));
// A kind-only scope confines to that kind, at any path.
let kind_only = job_read_run_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap();
assert!(run_confinement_admits(
&kind_only,
"scripts",
"u/admin/anything"
));
assert!(!run_confinement_admits(&kind_only, "flows", "f/team/etl"));
let kind_only = job_read_confinement(Some(&["jobs:run:scripts".to_string()])).unwrap();
assert!(kind_only.admits("scripts", "u/admin/anything"));
assert!(!kind_only.admits("flows", "f/team/etl"));
// Scopes that grant job reads in their own right leave reads unconfined.
for scopes in [
@@ -1171,11 +1305,111 @@ mod tests {
vec!["if_jobs:filter_tags:deno".to_string()],
] {
assert!(
job_read_run_confinement(Some(&scopes)).is_none(),
job_read_confinement(Some(&scopes)).is_none(),
"{scopes:?} must not confine job reads"
);
}
assert!(job_read_run_confinement(None).is_none());
assert!(job_read_confinement(None).is_none());
}
#[test]
fn jobs_read_paths_confine_reads_and_reach_only_confined_routes() {
let scopes = |s: &[&str]| s.iter().map(|s| s.to_string()).collect::<Vec<_>>();
let scoped = scopes(&["jobs:read:f/served/*,u/svc/report"]);
let confinement = job_read_confinement(Some(&scoped)).unwrap();
// Any kind of runnable at a named path.
assert!(confinement.admits("flows", "f/served/etl"));
assert!(confinement.admits("scripts", "u/svc/report"));
assert!(!confinement.admits("scripts", "f/other/etl"));
assert!(!confinement.admits("scripts", "f/served_not/etl"));
let lists = |s: &[&str], path: &str| job_list_path_filter(Some(&scopes(s))).allows(path);
assert!(lists(
&["jobs:read:f/served/*,u/svc/report"],
"f/served/etl"
));
assert!(!lists(
&["jobs:read:f/served/*,u/svc/report"],
"f/other/etl"
));
// A run scope adds the runnables it may start to the by-id reads, and nothing to
// the lists, not even a bare one that leaves the by-id reads unconfined.
let both = scopes(&["jobs:read:f/served/*", "jobs:run:scripts:f/other/etl"]);
let confinement = job_read_confinement(Some(&both)).unwrap();
assert!(confinement.admits("scripts", "f/other/etl"));
assert!(!confinement.admits("flows", "f/other/etl"));
for run in ["jobs:run:scripts:f/other/etl", "jobs:run"] {
assert!(
!lists(&["jobs:read:f/served/*", run], "f/other/etl"),
"{run}"
);
}
for unfiltered in [
&["jobs:read"][..],
&["jobs:read:f/served/*", "jobs:write"],
&["if_jobs:filter_tags:deno"],
] {
assert!(lists(unfiltered, "f/other/etl"), "{unfiltered:?}");
}
for unconfined in [
&["jobs:read:*"][..],
&["jobs:read:f/served/*", "jobs:read"],
&["jobs:read:f/served/*", "jobs:write"],
&["jobs:write:f/served/*"],
] {
assert!(
job_read_confinement(Some(&scopes(unconfined))).is_none(),
"{unconfined:?}"
);
}
let id = "0190f4c2-0000-7000-8000-000000000000";
for route in [
format!("jobs_u/get/{id}"),
format!("jobs_u/completed/get_result/{id}"),
format!("jobs_u/getupdate_sse/{id}"),
format!("jobs_u/flow/approval_info/{id}"),
"jobs/list".to_string(),
"jobs/completed/list".to_string(),
"jobs/queue/list".to_string(),
] {
let route = format!("/api/w/ws/{route}");
assert!(
check_route_access(&scoped, &route, "GET").is_ok(),
"{route}"
);
}
// Reads that aggregate over the workspace, resolve past the job they name, or
// mint a credential stay with a `jobs:read` that names no path.
for route in [
"jobs/list_filtered_uuids".to_string(),
"jobs/queue/list_filtered_uuids".to_string(),
"jobs/queue/count".to_string(),
"jobs/completed/count".to_string(),
"jobs/completed/count_jobs".to_string(),
"jobs/completed/export".to_string(),
"jobs/queue/export".to_string(),
"jobs/asset_dispatch_edges".to_string(),
format!("jobs/queue/scheduled_for/{id}"),
format!("jobs/result_by_id/{id}/a"),
format!("jobs/job_view_token/{id}"),
format!("jobs_u/get_root_job_id/{id}"),
] {
let route = format!("/api/w/ws/{route}");
assert!(
check_route_access(&scoped, &route, "GET").is_err(),
"{route}"
);
assert!(
check_route_access(&scopes(&["jobs:read"]), &route, "GET").is_ok(),
"{route}"
);
assert!(
check_route_access(&scopes(&["jobs:read:*"]), &route, "GET").is_ok(),
"{route}"
);
}
}
#[test]
@@ -1585,6 +1819,6 @@ mod tests {
}
// A cancel scope neither confines nor frees a run token's reads.
let run = scopes(&["jobs:run:flows:f/a/b", "jobs:cancel"]);
assert!(job_read_run_confinement(Some(&run)).is_some());
assert!(job_read_confinement(Some(&run)).is_some());
}
}
backend/windmill-api-integration-tests/tests/workspace_comparison.rs
+11 -1
View File
@@ -756,6 +756,13 @@ async fn test_compare_workspaces_trigger_and_schedule(db: Pool<Postgres>) -> any
)
.execute(&db)
.await?;
// The scheduler's missed-occurrence counters only ever move on the enabled side.
sqlx::query(
"UPDATE schedule SET late_run_streak = 1, missed_occurrences = 2, last_missed_at = NOW()
WHERE workspace_id = 'test-workspace' AND path = 'f/sch/runtime_only'",
)
.execute(&db)
.await?;
// ------ Schedule: config change (script_path) in fork. Should diff.
sqlx::query!(
@@ -2537,7 +2544,10 @@ async fn test_rename_records_the_vacated_path(db: Pool<Postgres>) -> anyhow::Res
.await?;
let status = resp.status();
let hash = resp.text().await?;
assert!(status.is_success(), "script deploy failed: {status} — {hash}");
assert!(
status.is_success(),
"script deploy failed: {status} — {hash}"
);
parent_hash = Some(hash.trim().trim_matches('"').to_string());
}
let mut vacated = None;
backend/windmill-api-jobs/Cargo.toml
+1
View File
@@ -15,6 +15,7 @@ enterprise = ["windmill-common/enterprise"]
[dependencies]
windmill-api-auth.workspace = true
windmill-api-sse.workspace = true
windmill-audit.workspace = true
windmill-common = { workspace = true, default-features = false }
windmill-queue.workspace = true
axum.workspace = true
backend/windmill-api-jobs/src/execution.rs
+32 -4
View File
@@ -23,6 +23,10 @@ use std::sync::Arc;
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed};
use windmill_audit::{
audit_oss::{audit_log_many, AuditAuthorable},
ActionKind,
};
use windmill_common::{
db::{UserDB, UserDbWithAuthed},
error::{self, Error},
@@ -196,11 +200,15 @@ pub async fn check_license_key_valid() -> error::Result<()> {
pub async fn cancel_jobs(
jobs: Vec<Uuid>,
db: &DB,
username: &str,
author: &(impl AuditAuthorable + Sync),
w_id: &str,
force_cancel: bool,
) -> error::JsonResult<Vec<Uuid>> {
let username = author.username();
let mut uuids = vec![];
// A job that completed before its turn is reported back like the others but was not
// cancelled, so it gets no audit entry.
let mut audited = vec![];
tracing::info!("Cancelling jobs: {:?}", jobs);
let mut tx = db.begin().await?;
let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj
@@ -246,7 +254,7 @@ pub async fn cancel_jobs(
}
match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move {
let tx = db.begin().await?;
let (tx, _) = cancel_job(
let (tx, cancelled) = cancel_job(
username,
None,
job_id.clone(),
@@ -258,13 +266,14 @@ pub async fn cancel_jobs(
)
.await?;
tx.commit().await?;
Ok::<_, anyhow::Error>(())
Ok::<_, anyhow::Error>(cancelled)
})
.await
{
Ok(result) => match result {
Ok(_) => {
Ok(cancelled) => {
uuids.push(job_id);
audited.extend(cancelled);
}
Err(e) => {
tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e);
@@ -279,8 +288,27 @@ pub async fn cancel_jobs(
}
}
audited.extend(trivial_jobs.iter().copied());
uuids.extend(trivial_jobs);
// The cancels are committed by now: failing here would report them as not done.
if let Err(e) = audit_log_many(
db,
author,
if force_cancel {
"jobs.force_cancel"
} else {
"jobs.cancel"
},
ActionKind::Delete,
w_id,
&audited.iter().map(|id| id.to_string()).collect::<Vec<_>>(),
)
.await
{
tracing::error!("Failed to write audit entries for cancelled jobs in {w_id}: {e:#}");
}
Ok(Json(uuids))
}
backend/windmill-api-jobs/src/query.rs
+28
View File
@@ -26,6 +26,34 @@ fn not_in_nullable(col: &str, quoted: &[String]) -> String {
)
}
/// Narrows a job list to the top-level runs of deployed scripts and flows at an `exact`
/// path, or at or under a `prefix` on the `/` boundary (`ScopePathFilter::allows`).
///
/// Top-level only because every `runnable_path` index on `v2_job` is partial on
/// `parent_job IS NULL`. Deployed kinds only because a preview's `runnable_path` is
/// whatever its caller sent.
pub fn and_where_root_run_at_paths(sqlb: &mut SqlBuilder, exact: &[String], prefix: &[String]) {
let mut clauses = Vec::new();
let equal: Vec<_> = exact.iter().chain(prefix).map(|p| quote(p)).collect();
if !equal.is_empty() {
clauses.push(format!("v2_job.runnable_path IN ({})", equal.join(", ")));
}
for p in prefix {
let pattern = format!("{}/%", escape_ilike_pattern(p));
clauses.push(format!("v2_job.runnable_path LIKE {}", quote(&pattern)));
}
if clauses.is_empty() {
sqlb.and_where("false");
return;
}
sqlb.and_where_is_null("v2_job.parent_job")
.and_where(
"v2_job.kind IN ('script', 'script_hub', 'unassigned_script', 'flow', \
'unassigned_flow', 'singlestepflow', 'unassigned_singlestepflow')",
)
.and_where(format!("({})", clauses.join(" OR ")));
}
pub fn filter_list_queue_query(
mut sqlb: SqlBuilder,
lq: &ListQueueQuery,
backend/windmill-api-schedule/src/lib.rs
+3 -2
View File
@@ -1487,11 +1487,12 @@ async fn set_default_error_handler(
}
};
if let Some(value_content) = value {
windmill_api_settings::set_global_setting_internal(&db, key, value_content).await?;
if let Some(value_content) = value.clone() {
windmill_api_settings::set_global_setting_internal(&db, key.clone(), value_content).await?;
} else {
windmill_api_settings::delete_global_setting(&db, key.as_str()).await?;
}
windmill_api_settings::audit_setting_write(&db, &authed, &key, value.as_ref()).await?;
if payload.override_existing {
// The rewrite and its history rows go in one transaction: on separate
backend/windmill-api-settings/src/lib.rs
+234 -8
View File
@@ -29,6 +29,7 @@ mod log_cleanup;
mod storage_usage;
use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed};
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT;
use windmill_common::DB;
@@ -829,7 +830,71 @@ pub async fn set_global_setting(
Json(value): Json<Value>,
) -> error::Result<()> {
require_super_admin(&db, &authed).await?;
set_global_setting_internal(&db, key, value.value.unwrap_or(serde_json::Value::Null)).await
let value = value.value.unwrap_or(serde_json::Value::Null);
set_global_setting_internal(&db, key.clone(), value.clone()).await?;
audit_setting_write(&db, &authed, &key, Some(&value)).await
}
/// A setting is recorded by name, with its value only when that is a boolean or a number.
/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no
/// list of secret-bearing settings stays complete as settings are added.
///
/// AUTHORIZATION: records a write, checks nothing. The caller must have established that
/// `authed` may change instance settings before performing the write this records.
pub async fn audit_setting_write(
db: &DB,
authed: &ApiAuthed,
key: &str,
value: Option<&serde_json::Value>,
) -> error::Result<()> {
let deleted = match value {
None | Some(serde_json::Value::Null) => true,
Some(serde_json::Value::String(s)) => s.trim().is_empty(),
Some(_) => false,
};
let recorded = match value {
Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => {
Some(v.to_string())
}
_ => None,
};
audit_log(
db,
authed,
if deleted {
"global_settings.delete"
} else {
"global_settings.update"
},
if deleted {
ActionKind::Delete
} else {
ActionKind::Update
},
"global",
Some(key),
recorded.as_deref().map(|v| [("value", v)].into()),
)
.await
}
async fn audit_instance_action(
db: &DB,
authed: &ApiAuthed,
operation: &str,
action_kind: ActionKind,
parameters: Option<std::collections::HashMap<&str, &str>>,
) -> error::Result<()> {
audit_log(
db,
authed,
operation,
action_kind,
"global",
Some(&authed.email),
parameters,
)
.await
}
pub async fn set_global_setting_internal(
@@ -1305,15 +1370,61 @@ async fn set_instance_config(
run_setting_pre_write_hook(&db, key, value).await?;
}
}
// The external-instance write removes its own key from the diff.
let external = settings_diff
.upserts
.get(EXTERNAL_INSTANCE_PG_SETTING)
.cloned()
.map(Some)
.or_else(|| {
settings_diff
.deletes
.iter()
.any(|k| k == EXTERNAL_INSTANCE_PG_SETTING)
.then_some(None)
});
windmill_common::external_instance_pg::write_external_instance_pg_from_diff(
&db,
&mut settings_diff,
)
.await?;
if let Some(value) = external {
audit_setting_write(&db, &authed, EXTERNAL_INSTANCE_PG_SETTING, value.as_ref()).await?;
}
instance_config::apply_settings_diff(&db, &settings_diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
// Applied and audited one setting at a time: the writes are not transactional, so a
// batch that fails midway must not leave the settings it did write unrecorded.
let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } =
settings_diff;
let steps = upserts
.into_iter()
.map(|(k, v)| (k, Some(v)))
.chain(deletes.into_iter().map(|k| (k, None)))
.map(Some)
.chain(std::iter::once(None));
for step in steps {
let mut diff = instance_config::SettingsDiff::default();
match &step {
Some((key, value)) => {
if let Some(previous) = previous_values.remove(key) {
diff.previous_values.insert(key.clone(), previous);
}
match value {
Some(value) => {
diff.upserts.insert(key.clone(), value.clone());
}
None => diff.deletes.push(key.clone()),
}
}
None => diff.unchanged_count = unchanged_count,
}
instance_config::apply_settings_diff(&db, &diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
if let Some((key, value)) = &step {
audit_setting_write(&db, &authed, key, value.as_ref()).await?;
}
}
if ai_config_changed {
bump_instance_ai_config_revision();
@@ -1343,9 +1454,37 @@ async fn set_instance_config(
.collect();
let configs_diff =
instance_config::diff_worker_configs(&current_wc, &desired_wc, ApplyMode::Merge);
instance_config::apply_configs_diff(&db, &configs_diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
let steps = configs_diff
.upserts
.into_iter()
.map(|(g, c)| (g, Some(c)))
.chain(configs_diff.deletes.into_iter().map(|g| (g, None)));
for (group, config) in steps {
let mut diff = instance_config::ConfigsDiff::default();
let (operation, action_kind) = match config {
Some(config) => {
diff.upserts.insert(group.clone(), config);
("worker_config.update", ActionKind::Update)
}
None => {
diff.deletes.push(group.clone());
("worker_config.delete", ActionKind::Delete)
}
};
instance_config::apply_configs_diff(&db, &diff)
.await
.map_err(|e| error::Error::internal_err(e.to_string()))?;
audit_log(
&db,
&authed,
operation,
action_kind,
"global",
Some(&format!("worker__{group}")),
None,
)
.await?;
}
}
Ok(())
@@ -1583,6 +1722,14 @@ pub async fn renew_license_key(
}
)));
} else {
audit_instance_action(
&db,
&authed,
"settings.renew_license_key",
ActionKind::Update,
None,
)
.await?;
return Ok("Renewed license key".to_string());
}
}
@@ -1812,6 +1959,14 @@ async fn refresh_custom_instance_user_pwd(
require_super_admin(&db, &authed).await?;
windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?;
windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?;
audit_instance_action(
&db,
&authed,
"settings.refresh_custom_instance_user_pwd",
ActionKind::Update,
None,
)
.await?;
Ok(Json(()))
}
@@ -1970,7 +2125,8 @@ async fn setup_custom_instance_pg_database(
windmill_common::datatable_roles::lock_instance_databases_governance(&mut tx, [dbname.trim()])
.await?;
let mut logs = CustomInstanceDbLogs::default();
let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
let result =
setup_custom_instance_pg_database_inner(authed.clone(), &db, &dbname, &mut logs).await;
let success = result.is_ok();
let error = result.err().map(|e| e.to_string());
let status = CustomInstanceDb {
@@ -1998,6 +2154,20 @@ async fn setup_custom_instance_pg_database(
.fetch_one(&mut *tx)
.await?;
tx.commit().await?;
audit_instance_action(
&db,
&authed,
"settings.setup_custom_instance_pg_database",
ActionKind::Create,
Some(
[
("dbname", dbname.as_str()),
("success", if success { "true" } else { "false" }),
]
.into(),
),
)
.await?;
let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?;
Ok(Json(status))
@@ -2176,6 +2346,14 @@ async fn drop_custom_instance_pg_database(
require_super_admin(&db, &authed).await?;
windmill_common::drop_custom_instance_database(&db, &dbname).await?;
audit_instance_action(
&db,
&authed,
"settings.drop_custom_instance_pg_database",
ActionKind::Delete,
Some([("dbname", dbname.as_str())].into()),
)
.await?;
Ok(format!("Database '{}' dropped successfully", dbname))
}
@@ -2220,6 +2398,14 @@ pub async fn migrate_secrets_to_vault(
let report = windmill_common::secret_backend::migrate_secrets_to_vault(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_to_vault",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
@@ -2241,6 +2427,14 @@ pub async fn migrate_secrets_to_database(
let report =
windmill_common::secret_backend::migrate_secrets_to_database(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_to_database",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
@@ -2274,6 +2468,14 @@ pub async fn migrate_secrets_to_azure_kv(
let report =
windmill_common::secret_backend::migrate_secrets_to_azure_kv(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_to_azure_kv",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
@@ -2291,6 +2493,14 @@ pub async fn migrate_secrets_from_azure_kv(
let report =
windmill_common::secret_backend::migrate_secrets_from_azure_kv(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_from_azure_kv",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
@@ -2315,6 +2525,14 @@ pub async fn migrate_secrets_to_aws_sm(
) -> JsonResult<SecretMigrationReport> {
require_super_admin(&db, &authed).await?;
let report = windmill_common::secret_backend::migrate_secrets_to_aws_sm(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_to_aws_sm",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
@@ -2328,6 +2546,14 @@ pub async fn migrate_secrets_from_aws_sm(
require_super_admin(&db, &authed).await?;
let report =
windmill_common::secret_backend::migrate_secrets_from_aws_sm(&db, &settings).await?;
audit_instance_action(
&db,
&authed,
"settings.migrate_secrets_from_aws_sm",
ActionKind::Update,
None,
)
.await?;
Ok(Json(report))
}
backend/windmill-api-workspaces/src/workspaces.rs
+13 -4
View File
@@ -744,13 +744,15 @@ async fn is_premium(
#[derive(Serialize)]
struct BillableSeatsResponse {
/// Both omitted when the seats counted are another workspace's: a fork member need not be a
/// All three omitted when the seats counted are another workspace's: a fork member need not be a
/// member of the billing root, so the root's headcount is not theirs to read. The total is,
/// since it is the divisor of the quota their own executions draw on.
#[serde(skip_serializing_if = "Option::is_none")]
developers: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
operators: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
service_accounts: Option<i64>,
seats: i64,
}
@@ -780,6 +782,7 @@ async fn get_billable_seats(
Ok(Json(BillableSeatsResponse {
developers: own.then_some(counted.developers),
operators: own.then_some(counted.operators),
service_accounts: own.then_some(counted.service_accounts),
seats: counted.seats,
}))
}
@@ -10471,7 +10474,7 @@ async fn edit_workspace(
pub(crate) async fn archive_workspace_impl(
db: &DB,
w_id: &str,
username: &str,
authed: &ApiAuthed,
// When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the
// prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the
// archive — a later failure can't strand a half-archived dev that's still flagged/locked.
@@ -10585,7 +10588,7 @@ pub(crate) async fn archive_workspace_impl(
let axum::Json(canceled_jobs) = windmill_api_jobs::cancel_jobs(
jobs_to_cancel,
db,
username,
authed,
w_id,
false, // force_cancel
)
@@ -10645,7 +10648,7 @@ async fn archive_workspace(
// dev workspace, which only applies to a workspace that is itself a dev (`dev_lock_parent`): a
// root archived out from under its dev is the pre-existing shape and not this pairing's to police.
let (schedules_count, canceled_count, deleted_tokens_count) =
archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?;
archive_workspace_impl(&db, &w_id, &authed, dev_lock_parent.as_deref()).await?;
// Audit log
let mut tx = db.begin().await?;
@@ -14008,6 +14011,12 @@ const TRIGGER_COMPARE_IGNORE: &[&str] = &[
"last_server_ping",
"extra_perms",
"permissioned_as",
// Schedule counters the scheduler bumps as occurrences are missed. A fork's
// schedules are disabled so theirs stay at the defaults, and no deploy carries
// them across: compared, a parent schedule that ever ran late differs forever.
"late_run_streak",
"missed_occurrences",
"last_missed_at",
// Server-managed fields that the merge feature treats as workspace-local
// (regenerated by the deploy handler): GCP `subscription_id` is rewritten
// to `windmill_<workspace_id>_<path>` in `CreateUpdate` mode, and Azure's
backend/windmill-api-workspaces/src/workspaces_extra.rs
+1 -1
View File
@@ -944,7 +944,7 @@ pub(crate) async fn change_workspace_id(
// Note: schedules were already moved to new workspace, so this will find 0 schedules
info!("Archiving old workspace");
let (_schedules_count, canceled_count, _deleted_tokens_count) =
archive_workspace_impl(&db, &old_id, &authed.username, None).await?;
archive_workspace_impl(&db, &old_id, &authed, None).await?;
// The old id stays live between the commit above and the archive, and a fork copy created for
// it in that window registers under it. Creation checks the workspace is live under the fork
backend/windmill-api/openapi-deref.json
+4
View File
@@ -5424,6 +5424,10 @@
"is_past_due": {
"type": "boolean"
},
"is_canceled": {
"type": "boolean",
"description": "the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions"
},
"max_tolerated_executions": {
"type": "number"
}
backend/windmill-api/openapi-deref.yaml
+3
View File
@@ -5286,6 +5286,9 @@ paths:
type: string
is_past_due:
type: boolean
is_canceled:
type: boolean
description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions
max_tolerated_executions:
type: number
required:
backend/windmill-api/openapi.yaml
+41 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.823.1
version: 1.826.0
title: Windmill API
contact:
@@ -4492,6 +4492,12 @@ paths:
description: >-
Omitted when the seats counted are another workspace's, as they are for a
fork resolving to its billing root.
service_accounts:
type: integer
description: >-
Enabled service accounts, half a seat each like operators. Omitted when the
seats counted are another workspace's, as they are for a fork resolving to
its billing root.
seats:
type: integer
required:
@@ -4529,6 +4535,9 @@ paths:
type: string
is_past_due:
type: boolean
is_canceled:
type: boolean
description: the team plan was canceled this month; it stays premium until the month ends, capped at max_tolerated_executions
max_tolerated_executions:
type: number
required:
@@ -16574,6 +16583,31 @@ paths:
required:
- database_length
/w/{workspace}/jobs/oldest:
get:
summary: get the creation date of the oldest job
operationId: getOldestJob
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: all_workspaces
description: consider jobs from all workspaces (only valid if request come from the `admins` workspace)
in: query
schema:
type: boolean
responses:
"200":
description: creation date of the oldest job, absent when there is none
content:
application/json:
schema:
type: object
properties:
created_at:
type: string
format: date-time
/w/{workspace}/jobs/completed/count:
get:
summary: get completed count
@@ -28534,6 +28568,8 @@ components:
$ref: "../../openflow.openapi.yaml#/components/schemas/AiTransform"
ProviderConfig:
$ref: "../../openflow.openapi.yaml#/components/schemas/ProviderConfig"
ProviderTransform:
$ref: "../../openflow.openapi.yaml#/components/schemas/ProviderTransform"
MemoryConfig:
$ref: "../../openflow.openapi.yaml#/components/schemas/MemoryConfig"
FlowModuleValue:
@@ -28554,6 +28590,8 @@ components:
$ref: "../../openflow.openapi.yaml#/components/schemas/BranchAll"
AiAgent:
$ref: "../../openflow.openapi.yaml#/components/schemas/AiAgent"
AiDecision:
$ref: "../../openflow.openapi.yaml#/components/schemas/AiDecision"
Identity:
$ref: "../../openflow.openapi.yaml#/components/schemas/Identity"
FlowStatus:
@@ -29518,6 +29556,8 @@ components:
togetherai,
aws_bedrock,
customai,
typesafe,
cloudflare,
]
GitSyncObjectType:
backend/windmill-api/src/agent_runs.rs
+1 -1
View File
@@ -29,7 +29,7 @@ use crate::db::{ApiAuthed, DB};
use crate::jobs::{handle_chat_conversation_messages, set_flow_memory_id, RunJobQuery};
/// The id the agent's step carries in the run, as the editor's chat names it. A token scoped to
/// `jobs:run:agents:<path>` reads its runs back by it (`require_job_within_run_scope`).
/// `jobs:run:agents:<path>` reads its runs back by it (`require_job_within_read_scope`).
const AGENT_NODE_ID: &str = "__wm_agent_root";
/// A turn of the agent's chat when `memory_id` names a conversation, a single run otherwise.
backend/windmill-api/src/ai.rs
+22 -4
View File
@@ -246,6 +246,12 @@ struct AIStandardResource {
/// Custom HTTP headers to include in AI requests
#[serde(default)]
headers: HashMap<String, String>,
/// The API token of a `cloudflare` resource, which names it `token` rather than `api_key`.
#[serde(default, deserialize_with = "empty_string_as_none")]
token: Option<String>,
/// The account of a `cloudflare` resource, which its Workers AI URL is built from.
#[serde(default, deserialize_with = "empty_string_as_none")]
account_id: Option<String>,
}
#[derive(Deserialize, Debug)]
@@ -293,10 +299,21 @@ async fn resolve_provider_credentials(
// Skip get_base_url for Bedrock - it uses SDK directly, not HTTP
let base_url = if matches!(provider, AIProvider::AWSBedrock) {
String::new()
} else if *provider == AIProvider::Cloudflare && resource.base_url.is_none() {
let account_id = if let Some(account_id) = resource.account_id {
Some(resolve_var(account_id, db, w_id, user_db.as_ref(), authed).await?)
} else {
None
};
windmill_ai::ai_providers::cloudflare_workers_ai_base_url(account_id.as_deref())?
} else {
provider.get_base_url(resource.base_url, db).await?
};
let api_key = if let Some(api_key) = resource.api_key {
let api_key = match provider {
AIProvider::Cloudflare => resource.api_key.or(resource.token),
_ => resource.api_key,
};
let api_key = if let Some(api_key) = api_key {
Some(resolve_var(api_key, db, w_id, user_db.as_ref(), authed).await?)
} else {
None
@@ -486,9 +503,10 @@ async fn assume_bedrock_role(
) -> Result<windmill_ai::ai_bedrock::AssumedRoleCredentials> {
#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))]
{
use windmill_common::oidc_oss::{generate_id_token, WorkspaceClaim};
use windmill_common::oidc_oss::{generate_id_token, OidcCaller, WorkspaceClaim};
let id_token = generate_id_token(
OidcCaller::Bedrock,
Some(db),
WorkspaceClaim { workspace: w_id.to_string() },
windmill_ai::ai_bedrock::AWS_OIDC_AUDIENCE,
@@ -1238,8 +1256,8 @@ async fn proxy(
.get("X-Resource-Value")
.map(|v| {
let invalid = |e: String| Error::BadRequest(format!("Invalid X-Resource-Value: {e}"));
let decoded =
urlencoding::decode(v.to_str().unwrap_or("")).map_err(|e| invalid(e.to_string()))?;
let decoded = urlencoding::decode(v.to_str().unwrap_or(""))
.map_err(|e| invalid(e.to_string()))?;
let value = serde_json::from_str::<serde_json::Value>(&decoded)
.map_err(|e| invalid(e.to_string()))?;
check_scopes(&authed, || "resources:write".to_string())?;
backend/windmill-api/src/jobs.rs
+134 -36
View File
@@ -292,6 +292,7 @@ pub fn workspaced_service() -> Router {
)
.route("/run/dynamic_select", post(run_dynamic_select))
.route("/list", get(list_jobs))
.route("/oldest", get(get_oldest_job))
.route("/asset_dispatch_edges", get(list_asset_dispatch_edges))
.route(
"/list_selected_job_groups",
@@ -497,6 +498,28 @@ async fn get_result_by_id(
view_token.as_deref(),
)
.await?;
// The node is resolved up through the enclosing flows, so a read path that admitted
// `flow_id` as a step must also admit the run it is a step of. A path-scoped
// `jobs:read` does not reach this route; a `jobs:run` scope beside it does.
if windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref())
.is_some_and(|c| c.has_read_paths())
{
let top = sqlx::query_scalar::<_, Uuid>(
"WITH RECURSIVE chain(id, parent_job) AS (
SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2
UNION ALL
SELECT j.id, j.parent_job FROM v2_job j
JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2
)
SELECT id FROM chain WHERE parent_job IS NULL",
)
.bind(flow_id)
.bind(&w_id)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("Job {flow_id} not found")))?;
require_job_within_read_scope(&db, &authed, &w_id, &top).await?;
}
let res =
windmill_queue::get_result_by_id(db.clone(), w_id.clone(), flow_id, node_id, json_path)
@@ -1728,7 +1751,7 @@ async fn require_job_access(
job_id: &Uuid,
created_by: &str,
view_token: Option<&str>,
run_confined: bool,
read_confined: bool,
) -> error::Result<()> {
// Tag scope (`if_jobs:filter_tags:`) is an orthogonal hard restriction on a
// scoped token: it must never read a job outside its allowed tags, regardless of
@@ -1753,11 +1776,11 @@ async fn require_job_access(
}
}
// A path-scoped `jobs:run` token is likewise hard-restricted to the runnables it
// may start, ahead of every grant below — the token is handed out to run one thing,
// so it must not read jobs of anything else merely because its owner could.
if run_confined {
require_job_within_run_scope(db, authed, w_id, job_id).await?;
// A path-scoped `jobs:run` or `jobs:read` token is likewise hard-restricted to the
// runnables it names, ahead of every grant below — the token is handed out for one
// thing, so it must not read jobs of anything else merely because its owner could.
if read_confined {
require_job_within_read_scope(db, authed, w_id, job_id).await?;
}
// Fast path: you can always read a job you launched. This is also load-bearing
@@ -1906,20 +1929,35 @@ async fn require_job_access(
/// could start an app's inline-script component but not read the run back — those jobs
/// are `AppScript`/`Preview` kinds that no `jobs:run` scope can name.
///
/// No-op — and no query — for every caller whose job reads are not run-confined (see
/// `job_read_run_confinement`), which is all sessions, unscoped tokens and `jobs:read`
/// tokens.
async fn require_job_within_run_scope(
/// A `jobs:read:<paths>` scope confines the same way, by path alone: the job, or one of
/// its ancestors, must be a run of a deployed script, flow or agent at a named path.
/// Only those kinds count, because a preview's `runnable_path` is whatever its caller
/// sent and would otherwise let any preview pass for a named runnable.
///
/// No-op — and no query — for every caller whose job reads are not confined (see
/// `job_read_confinement`), which is all sessions, unscoped tokens and tokens whose
/// `jobs:read` names no path.
async fn require_job_within_read_scope(
db: &DB,
authed: &ApiAuthed,
w_id: &str,
job_id: &Uuid,
) -> error::Result<()> {
let Some(confinement) =
windmill_api_auth::scopes::job_read_run_confinement(authed.scopes.as_deref())
windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref())
else {
return Ok(());
};
// A job whose token is restricted by `job_token_scopes` still reads its own flow run
// (step results, user state), which the route check lets through whatever the
// scopes. A read path names what else it may read, not a fence around its own run.
if let Some(own_job) = authed.job_id {
if confinement.has_read_paths()
&& windmill_api_auth::auth::job_in_same_flow_run(db, own_job, *job_id).await
{
return Ok(());
}
}
// `scope_kind` is the runnable kind a `jobs:run:<kind>:<path>` scope can name, or
// NULL for a job no such scope reaches directly (previews, dependency jobs,
// flow-inlined scripts) — those are still readable as a step of a matching flow,
@@ -1967,13 +2005,7 @@ async fn require_job_within_run_scope(
let in_scope =
chain.iter().any(
|job| match (job.runnable_path.as_deref(), job.scope_kind.as_deref()) {
(Some(runnable_path), Some(kind))
if windmill_api_auth::scopes::run_confinement_admits(
&confinement,
kind,
runnable_path,
) =>
{
(Some(runnable_path), Some(kind)) if confinement.admits(kind, runnable_path) => {
true
}
_ => job.launched_by_app.as_deref().is_some_and(&runs_app),
@@ -1995,7 +2027,7 @@ async fn require_job_within_run_scope(
/// is within a scope on the flow. Only those kinds count, because a preview's
/// `runnable_path` is whatever its caller sent and would otherwise let any preview
/// impersonate an in-scope runnable. Agent runs are previews filed under the agent's
/// path, recognized the same way `require_job_within_run_scope` does.
/// path, recognized the same way `require_job_within_read_scope` does.
async fn filter_jobs_within_cancel_scope(
db: &DB,
authed: &ApiAuthed,
@@ -2269,7 +2301,7 @@ async fn require_job_cancel_access(
.fetch_optional(db)
.await?
.ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?;
let run_confined = !windmill_api_auth::scopes::has_job_cancel_grant(authed.scopes.as_deref());
let read_confined = !windmill_api_auth::scopes::has_job_cancel_grant(authed.scopes.as_deref());
require_job_access(
db,
user_db,
@@ -2278,7 +2310,7 @@ async fn require_job_cancel_access(
job_id,
&created_by,
None,
run_confined,
read_confined,
)
.await?;
require_job_within_cancel_scope(db, authed, w_id, *job_id).await
@@ -2414,7 +2446,7 @@ async fn get_job(
// an approval link for a job must not let a scoped token read one outside the
// runnables it may start.
if let Some(authed) = opt_authed.as_ref() {
require_job_within_run_scope(&db, authed, &w_id, &id).await?;
require_job_within_read_scope(&db, authed, &w_id, &id).await?;
}
} else {
require_opt_authed_job_read_access(
@@ -3332,6 +3364,7 @@ fn flow_tree_entry_label(
"forloopflow" => " forloop",
"whileloopflow" => " whileloop",
"aiagent" => " ai-agent",
"aidecision" => " ai-decision",
_ => "",
};
@@ -4388,6 +4421,17 @@ struct ListableQueuedJob {
pub workspace_id: String,
}
/// Narrows a job list to what a path-scoped `jobs:read:<paths>` token may read: the
/// top-level runs of the runnables it names. A step is read by id, under its flow. No-op
/// for every caller whose lists are not path-filtered (see `job_list_path_filter`).
fn confine_list_to_read_scope(sqlb: &mut SqlBuilder, authed: &ApiAuthed) {
if let windmill_api_auth::ScopePathFilter::Restricted { exact, prefix } =
windmill_api_auth::scopes::job_list_path_filter(authed.scopes.as_deref())
{
and_where_root_run_at_paths(sqlb, &exact, &prefix);
}
}
async fn list_queue_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -4409,7 +4453,7 @@ async fn list_queue_jobs(
"null as args"
};
let sql = list_queue_jobs_query(
let mut sqlb = list_queue_jobs_query(
&w_id,
&lq,
&[
@@ -4439,8 +4483,9 @@ async fn list_queue_jobs(
pagination,
false,
get_scope_tags(&authed),
)
.sql()?;
);
confine_list_to_read_scope(&mut sqlb, &authed);
let sql = sqlb.sql()?;
let mut tx = user_db.begin(&authed).await?;
let jobs = sqlx::query_as::<_, ListableQueuedJob>(&sql)
.fetch_all(&mut *tx)
@@ -4535,7 +4580,7 @@ async fn cancel_selection(
let Json(mut w_cancelled) = cancel_jobs(
ids,
&db,
authed.username.as_str(),
&authed,
workspace_id.as_str(),
force_cancel,
)
@@ -4727,6 +4772,41 @@ async fn count_completed_jobs(
))
}
#[derive(Deserialize)]
struct OldestJobQuery {
all_workspaces: Option<bool>,
}
#[derive(Serialize)]
struct OldestJob {
#[serde(skip_serializing_if = "Option::is_none")]
created_at: Option<chrono::DateTime<chrono::Utc>>,
}
/// Where the job history starts, whatever the caller may see of it: a client scanning the
/// runs backwards in time windows needs a bound that row-level visibility cannot make slow.
async fn get_oldest_job(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(q): Query<OldestJobQuery>,
) -> error::JsonResult<OldestJob> {
let created_at = if w_id == "admins" && q.all_workspaces.unwrap_or(false) {
// One index probe per workspace: a plain min() over the table has no index to walk.
sqlx::query_scalar(
"SELECT min(o.created_at) FROM workspace w CROSS JOIN LATERAL \
(SELECT min(created_at) AS created_at FROM v2_job WHERE workspace_id = w.id) o",
)
.fetch_one(&db)
.await?
} else {
sqlx::query_scalar("SELECT min(created_at) FROM v2_job WHERE workspace_id = $1")
.bind(&w_id)
.fetch_one(&db)
.await?
};
Ok(Json(OldestJob { created_at }))
}
lazy_static::lazy_static! {
/// 0 keeps the connection-wide statement_timeout.
static ref LIST_JOBS_STATEMENT_TIMEOUT_SECS: u64 =
@@ -4826,7 +4906,7 @@ async fn list_jobs(
}
let sqlc = if lq.running.is_none() {
Some(list_completed_jobs_query(
let mut sqlc = list_completed_jobs_query(
&w_id,
Some(per_page),
0,
@@ -4834,7 +4914,9 @@ async fn list_jobs(
cj_fields_ref,
true,
get_scope_tags(&authed),
))
);
confine_list_to_read_scope(&mut sqlc, &authed);
Some(sqlc)
} else {
None
};
@@ -4861,6 +4943,7 @@ async fn list_jobs(
true,
get_scope_tags(&authed),
);
confine_list_to_read_scope(&mut sqlq, &authed);
if let Some(sqlc) = sqlc {
format!("{} UNION ALL {}", &sqlq.subquery()?, &sqlc.subquery()?,)
@@ -5284,6 +5367,11 @@ async fn get_approval_info(
"Must be logged in or provide a valid approval token".to_string(),
));
}
// Nothing below goes through `require_job_read_access`, and an approval token is no
// way around a scoped token's confinement.
if let Some(authed) = opt_authed.as_ref() {
require_job_within_read_scope(&db, authed, &w_id, &job_id).await?;
}
// Fetch job info
#[derive(sqlx::FromRow)]
@@ -5516,8 +5604,17 @@ async fn get_approval_info(
// Possession of view rights over this approval is sufficient to mint a
// share-read-link token for the flow: it only grants read (no resume), and only to
// an authenticated workspace member, so it never widens what the approver can do.
let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?;
let view_token = Some(format!("{}.{hmac}", row.id));
// Not for a token whose reads are confined: the link would outlive it and serve any
// member, which is why `job_view_token` is refused to it too.
let confined = opt_authed.as_ref().is_some_and(|authed| {
windmill_api_auth::scopes::job_read_confinement(authed.scopes.as_deref()).is_some()
});
let view_token = if confined {
None
} else {
let hmac = generate_view_token(&w_id, row.id, VIEW_TOKEN_DOMAIN, &db).await?;
Some(format!("{}.{hmac}", row.id))
};
Ok(Json(ApprovalInfo {
flow_id: row.id,
@@ -5979,7 +6076,7 @@ pub async fn get_suspended_job_flow(
// resume secret must not let a scoped token read a flow it may not run. Anonymous
// approvers are unaffected.
if let Some(authed) = authed.as_ref() {
require_job_within_run_scope(&db, authed, &w_id, &flow_id).await?;
require_job_within_read_scope(&db, authed, &w_id, &flow_id).await?;
}
let flow = GetQuery::new()
@@ -6163,7 +6260,7 @@ pub async fn get_flow_user_state(
// Reachable by a `jobs:run` token (it is one of the by-id routes a run needs), so
// apply the same run-scope confinement as the other single-job reads. RLS below
// still governs which jobs the owner's identity can see at all.
require_job_within_run_scope(&db, &authed, &w_id, &job_id).await?;
require_job_within_read_scope(&db, &authed, &w_id, &job_id).await?;
let mut tx = user_db.begin(&authed).await?;
let r = sqlx::query_scalar!(
r#"
@@ -11392,7 +11489,7 @@ async fn list_completed_jobs(
"null as args"
};
let sql = list_completed_jobs_query(
let mut sqlb = list_completed_jobs_query(
&w_id,
Some(per_page),
offset,
@@ -11432,8 +11529,9 @@ async fn list_completed_jobs(
],
false,
get_scope_tags(&authed),
)
.sql()?;
);
confine_list_to_read_scope(&mut sqlb, &authed);
let sql = sqlb.sql()?;
let mut tx = user_db.begin(&authed).await?;
let jobs = sqlx::query_as::<_, ListableCompletedJob>(&sql)
.fetch_all(&mut *tx)
@@ -11612,7 +11710,7 @@ async fn get_completed_job_result(
// confinement that gate carries — re-apply it, as `get_job` does for the
// approval token. Anonymous approval access is untouched.
if let Some(authed) = opt_authed.as_ref() {
require_job_within_run_scope(&db, authed, &w_id, &id).await?;
require_job_within_read_scope(&db, authed, &w_id, &id).await?;
}
} else {
require_opt_authed_job_read_access(
backend/windmill-api/src/static_assets.rs
+15
View File
@@ -100,6 +100,21 @@ fn serve_path(path: &str, original_path: &str, query: Option<&str>) -> Response<
res = res.header("X-Robots-Tag", "noindex, nofollow");
}
// The raw-app preview shell evaluates whatever js is posted to it, so it is
// only ever served opaque-origin, with the deployed app wrapper's exact flags
// (`get_raw_app_data`). Matched on the embedded file, not the request path,
// so `//ui_builder/...` is covered. A sandboxed app's editor preview loads it
// as served, framed with the same flags (`RAW_APP_SANDBOX_FLAGS` in the
// frontend); an unsandboxed one loads a same-origin blob: copy.
if path == "ui_builder/app-preview.html" {
res = res.header(
header::CONTENT_SECURITY_POLICY,
"sandbox allow-scripts allow-forms allow-popups \
allow-popups-to-escape-sandbox allow-downloads allow-modals \
allow-top-navigation",
);
}
// Add Content-Security-Policy header for static assets when policy is set
if !CSP_POLICY.is_empty() {
if let Ok(header_value) = HeaderValue::try_from(CSP_POLICY.as_str()) {
backend/windmill-api/src/token.rs
+1 -1
View File
@@ -193,7 +193,7 @@ lazy_static! {
ScopeOption {
value: "jobs:read".to_string(),
label: "Read".to_string(),
requires_resource_path: false,
requires_resource_path: true,
},
ScopeOption {
value: "jobs:write".to_string(),
backend/windmill-audit/Cargo.toml
+4 -1
View File
@@ -20,4 +20,7 @@ chrono.workspace = true
serde_json.workspace = true
tracing.workspace = true
windmill-common = { workspace = true, default-features = false }
lazy_static.workspace = true
lazy_static.workspace = true
[dev-dependencies]
tracing-subscriber.workspace = true
backend/windmill-audit/src/audit_oss.rs
+15
View File
@@ -88,6 +88,21 @@ pub async fn audit_log<'c, E: sqlx::Executor<'c, Database = Postgres>>(
Ok(())
}
/// AUTHORIZATION: records entries, checks nothing. The caller must have authorized `author`
/// for the operation on every resource in `w_id` before calling.
#[cfg(not(feature = "private"))]
pub async fn audit_log_many<'c, E: sqlx::Executor<'c, Database = Postgres>>(
_db: E,
_author: &impl AuditAuthorable,
_operation: &str,
_action_kind: ActionKind,
_w_id: &str,
_resources: &[String],
) -> Result<()> {
// Implementation is not open source as Audit logs is a Windmill Enterprise Edition feature
Ok(())
}
#[cfg(not(feature = "private"))]
pub async fn list_audit(
_tx: Transaction<'_, Postgres>,
backend/windmill-common/src/client.rs
+107 -1
View File
@@ -307,12 +307,13 @@ impl AuthedClient {
workspace_id: &str,
file_key: &str,
storage: Option<String>,
max_bytes: Option<usize>,
) -> anyhow::Result<bytes::Bytes> {
let mut query = vec![("file_key", file_key.to_string())];
if let Some(storage) = storage {
query.push(("storage", storage));
}
let response = self
let mut response = self
.force_client
.as_ref()
.unwrap_or(&HTTP_CLIENT)
@@ -331,6 +332,39 @@ impl AuthedClient {
.context("Failed to send download_s3_file request")
.map_err(|e| anyhow::anyhow!(e.to_string()))?;
if let Some(max_bytes) = max_bytes {
let size_error = || {
anyhow::anyhow!(
"S3 file '{file_key}' exceeds maximum supported download size of {max_bytes} bytes"
)
};
if response
.content_length()
.is_some_and(|len| len > max_bytes as u64)
{
return Err(size_error());
}
let status = response.status();
let mut bytes = Vec::new();
// Content-Length may be absent (including for decoded compressed responses).
while let Some(chunk) = response
.chunk()
.await
.context("Failed to read response bytes")?
{
if chunk.len() > max_bytes - bytes.len() {
return Err(size_error());
}
bytes.extend_from_slice(&chunk);
}
return if status == reqwest::StatusCode::OK {
Ok(bytes.into())
} else {
Err(anyhow::anyhow!(String::from_utf8_lossy(&bytes).into_owned()))
};
}
match response.status().as_u16() {
200u16 => Ok(response
.bytes()
@@ -341,6 +375,78 @@ impl AuthedClient {
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
async fn download_response(
response: &'static str,
max_bytes: Option<usize>,
) -> anyhow::Result<bytes::Bytes> {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let client = AuthedClient::new(
format!("http://{}", listener.local_addr().unwrap()),
"test_ws".to_string(),
"test_token".to_string(),
Some(reqwest::Client::builder().no_proxy().build().unwrap()),
);
let server = tokio::spawn(async move {
let (socket, _) = listener.accept().await.unwrap();
let mut socket = BufReader::new(socket);
loop {
let mut line = String::new();
assert!(socket.read_line(&mut line).await.unwrap() > 0);
if line == "\r\n" {
break;
}
}
socket
.get_mut()
.write_all(response.as_bytes())
.await
.unwrap();
// Keep incomplete bodies open so rejection must happen before EOF.
std::future::pending::<()>().await;
});
let result = tokio::time::timeout(
Duration::from_secs(5),
client.download_s3_file("test_ws", "image.png", None, max_bytes),
)
.await;
server.abort();
result.expect("download waited for the oversized body to finish")
}
#[tokio::test]
async fn download_s3_file_rejects_oversized_body_before_eof() {
for response in [
"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n",
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n1\r\ne\r\n",
"HTTP/1.1 500 Internal Server Error\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nerror\r\n",
] {
let error = download_response(response, Some(4)).await.unwrap_err();
assert!(error.to_string().contains(
"S3 file 'image.png' exceeds maximum supported download size of 4 bytes"
));
}
}
#[tokio::test]
async fn download_s3_file_accepts_body_at_limit() {
for response in [
"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nabcd",
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nabcd\r\n0\r\n\r\n",
] {
assert_eq!(
download_response(response, Some(4)).await.unwrap(),
&b"abcd"[..]
);
}
}
}
#[inline]
fn query_from_json_path(json_path: Option<String>) -> Vec<(&'static str, String)> {
json_path
backend/windmill-common/src/flows.rs
+2
View File
@@ -357,6 +357,8 @@ fn check_module_value_is_composition_only(
}
}
}
// Runs no code: it sends its inputs to the decision model.
FlowModuleValue::AIDecision { tag, .. } => push_tag(tag),
}
Ok(())
}
backend/windmill-common/src/oidc_oss.rs
+15
View File
@@ -79,6 +79,8 @@ pub struct JobClaim {
pub tag: String,
pub worker_group: Option<String>,
pub run_as_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub job_token_scopes: Option<Vec<String>>,
pub jti: String,
}
@@ -87,13 +89,26 @@ lazy_static::lazy_static! {
static ref PRIVATE_KEY: RwLock<Option<String>> = RwLock::new(None);
}
#[cfg(not(feature = "private"))]
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum OidcCaller {
JobMint,
S3Instance,
S3Workspace,
Vault,
Sqs,
Bedrock,
}
#[cfg(not(feature = "private"))]
pub async fn generate_id_token<T: AdditionalClaims>(
_caller: OidcCaller,
_db: Option<&DB>,
_claim: T,
_audience: &str,
_identifier: String,
_email: Option<String>,
_expires_in: Option<i64>,
) -> Result<WindmillIdToken> {
Err(Error::internal_err(
"Not implemented in Windmill's Open Source repository".to_string(),
backend/windmill-common/src/otel_oss.rs
+6
View File
@@ -76,6 +76,9 @@ pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {}
#[cfg(not(feature = "private"))]
pub fn otel_incr_queue_push_count() {}
#[cfg(not(feature = "private"))]
pub fn otel_incr_oidc_signature_count(_caller: &'static str) {}
#[cfg(not(feature = "private"))]
pub fn otel_incr_queue_delete_count() {}
@@ -91,6 +94,9 @@ pub fn otel_incr_zombie_delete_count(_count: u64) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_queue_count(_tag: &str, _count: i64) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_queue_delay(_tag: &str, _delay_secs: f64) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_queue_running_count(_tag: &str, _count: i64) {}
backend/windmill-common/src/tracing_init.rs
+5
View File
@@ -44,6 +44,11 @@ lazy_static::lazy_static! {
/// Use `tracing::info!(target: windmill_common::tracing_init::VERBOSE_TARGET, ...)` for logs that should be suppressed in quiet mode.
pub const VERBOSE_TARGET: &str = "windmill_verbose";
/// Target of audit events, so a log pipeline can route them without matching on content.
/// It keeps the `windmill` prefix because that is what a `RUST_LOG` of `info` or `debug`
/// is rewritten to filter on.
pub const AUDIT_TARGET: &str = "windmill:audit";
/// Prefix used by user scripts to emit a log line as an OTEL tracing event
/// when `OTEL_JOB_LOGS=true`. Stripped before forwarding to the tracing layer.
pub const OTEL_PREFIX: &str = "OTEL: ";
backend/windmill-common/src/workspace_dependencies.rs
+1 -1
View File
@@ -510,7 +510,7 @@ impl WorkspaceDependenciesPrefetched {
// external.get(0).map(|wd| dbg!(wd.content.clone())).or(Some(
// "
// module mymod
// go 1.26
// go 1.27
// require ()
// "
// .to_owned(),
backend/windmill-common/src/workspaces.rs
+10 -5
View File
@@ -823,6 +823,7 @@ pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result<i64> {
pub struct BillableSeats {
pub developers: i64,
pub operators: i64,
pub service_accounts: i64,
pub seats: i64,
}
@@ -1043,9 +1044,10 @@ pub async fn guest_app_admits<'c, E: sqlx::Executor<'c, Database = sqlx::Postgre
Ok(admits.unwrap_or(false))
}
/// Billable members of `w_id` and the seats they cost, as `ceil(developers + operators/2)`. Service
/// accounts cannot log in and do not take a seat; a disabled member is not billed either. In a
/// workspace that granted operators builder rights, every operator counts as a developer.
/// Billable members of `w_id` and the seats they cost, as
/// `ceil(developers + (operators + service_accounts)/2)`. A service account is half a seat whatever
/// its role; a disabled member is not billed. In a workspace that granted operators builder rights,
/// every operator counts as a developer.
///
/// The workspace is invoiced by a job outside this codebase that counts the same rows with its own
/// SQL. The two must be changed together: this rule disagreeing with that one is what bills a
@@ -1057,7 +1059,8 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result<BillableSeats>
let row = sqlx::query!(
r#"SELECT
COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!",
COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS "operators!"
COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS "operators!",
COUNT(*) FILTER (WHERE is_service_account AND NOT disabled) AS "service_accounts!"
FROM usr WHERE workspace_id = $1"#,
w_id
)
@@ -1072,7 +1075,9 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result<BillableSeats>
Ok(BillableSeats {
developers,
operators,
seats: ((developers as f64) + 0.5 * (operators as f64)).ceil() as i64,
service_accounts: row.service_accounts,
seats: ((developers as f64) + 0.5 * ((operators + row.service_accounts) as f64)).ceil()
as i64,
})
}
backend/windmill-common/tests/billing_workspace.rs
+21 -10
View File
@@ -101,21 +101,27 @@ async fn billing_workspace_survives_cycles(db: Pool<Postgres>) {
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn paid_seats_and_fork_count(db: Pool<Postgres>) {
insert_ws(&db, "seat-root", None, false).await;
// 2 developers + 2 operators counted -> ceil(2 + 0.5*2) = 3.
insert_member(&db, "seat-root", "dev1@w.dev", false, false, false).await;
insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await;
insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await;
insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await;
// These must NOT count towards seats. The service account is a non-operator, so counting it
// would inflate the developer tally the invoice line is written from, not the operator one.
insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await;
// A service account is half a seat even with a developer role, counted apart from operators.
insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await;
// Not billed.
insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await;
insert_member(&db, "seat-root", "svc-disabled@w.dev", false, true, true).await;
assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3);
// 2 developers + (2 operators + 1 service account)/2 -> ceil(3.5) = 4.
assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 4);
let breakdown = billable_seats(&db, "seat-root").await.unwrap();
assert_eq!(
(breakdown.developers, breakdown.operators, breakdown.seats),
(2, 2, 3)
(
breakdown.developers,
breakdown.operators,
breakdown.service_accounts,
breakdown.seats
),
(2, 2, 1, 4)
);
sqlx::query(
@@ -128,9 +134,14 @@ async fn paid_seats_and_fork_count(db: Pool<Postgres>) {
windmill_common::workspaces::invalidate_operator_rights_cache("seat-root");
let breakdown = billable_seats(&db, "seat-root").await.unwrap();
assert_eq!(
(breakdown.developers, breakdown.operators, breakdown.seats),
(4, 0, 4),
"builder rights bill every operator as a developer"
(
breakdown.developers,
breakdown.operators,
breakdown.service_accounts,
breakdown.seats
),
(4, 0, 1, 5),
"builder rights bill every operator as a developer, service accounts stay half a seat"
);
insert_ws(&db, "seat-fork1", Some("seat-root"), false).await;
backend/windmill-queue/src/jobs.rs
+27 -4
View File
@@ -6261,9 +6261,20 @@ async fn push_inner<'c, 'd>(
if team_plan_status.max_tolerated_executions.is_none()
|| workspace_usage > team_plan_status.max_tolerated_executions.unwrap()
{
return Err(error::Error::QuotaExceeded(format!(
"Workspace {workspace_id} team plan is past due and isn't allowed to run any more jobs. Please fix your payment method in the workspace settings."
)));
// A canceled plan reuses the past-due cap for its last month. Only read
// here, on the refusal path, to word it.
let plan_canceled = sqlx::query_scalar::<_, bool>(
"SELECT COALESCE(plan = 'team_canceled', false) FROM workspace_settings WHERE workspace_id = $1",
)
.bind(&billing_w_id)
.fetch_optional(db)
.await?
.unwrap_or(false);
return Err(error::Error::QuotaExceeded(if plan_canceled {
format!("Workspace {workspace_id} team plan was canceled and has used the executions paid for this month. Jobs can run again on the free plan from the 1st, or right away by subscribing again in the workspace settings.")
} else {
format!("Workspace {workspace_id} team plan is past due and isn't allowed to run any more jobs. Please fix your payment method in the workspace settings.")
}));
}
} else {
if workspace_usage > MAX_FREE_EXECS
@@ -7512,6 +7523,18 @@ async fn push_inner<'c, 'd>(
concurrency_settings.concurrency_time_window_s,
)
};
// jsonb rejects the `\u0000` escape (22P05), which would fail the whole push, and args are
// not always what a caller typed: a flow step's are evaluated from the previous step's
// in-memory result, an agent tool's are written by the model.
let serialized_args = serde_json::value::to_raw_value(&args).map_err(|e| {
Error::internal_err(format!("Could not serialize args of job {job_id}: {e:#}"))
})?;
let sanitized_args = match strip_json_nul(serialized_args.get()) {
Cow::Owned(stripped) => RawValue::from_string(stripped).map_err(|e| {
Error::internal_err(format!("Could not sanitize args of job {job_id}: {e:#}"))
})?,
Cow::Borrowed(_) => serialized_args,
};
sqlx::query!(
"WITH inserted_job AS (
INSERT INTO v2_job (
@@ -7578,7 +7601,7 @@ async fn push_inner<'c, 'd>(
permissioned_as,
runnable_id,
runnable_path.clone(),
Json(args) as Json<PushArgs>,
Json(sanitized_args) as Json<Box<RawValue>>,
job_kind.clone() as JobKind,
trigger_path.flatten(),
language as Option<ScriptLang>,
backend/windmill-queue/tests/concurrency_counter_test.rs
+6 -19
View File
@@ -33,32 +33,19 @@ mod concurrency {
.unwrap_or(0)
}
/// Stress: N concurrent admit-attempts on a pre-existing key with limit
/// L < N. Asserts admitted count is exactly L (never over-admits) AND
/// that the row's job_uuids reflects exactly L entries.
/// Stress: N concurrent admit-attempts on a key with limit L < N. Asserts
/// admitted count is exactly L (never over-admits) AND that the row's
/// job_uuids reflects exactly L entries.
///
/// The row is pre-seeded with an empty `job_uuids` because there is a
/// known preexisting race when the `concurrency_counter` row does not
/// exist yet: `SELECT ... FOR UPDATE` on a missing row locks nothing,
/// so all concurrent transactions see `running=0` and all admit via
/// `INSERT ... ON CONFLICT DO UPDATE`. That race is inherited from
/// the legacy code path and is out of scope for this PR. This test
/// validates the property we *do* preserve: no over-admit once the
/// row exists.
/// The `concurrency_counter` row is deliberately absent at the start: the
/// monitor deletes empty rows while jobs for the key are still queued, so
/// the admit path has to serialize without a pre-existing row to lock.
#[sqlx::test(migrations = "../migrations")]
async fn stress_never_over_admits(db: Pool<Postgres>) {
let key = "stress-never-over-admits".to_string();
let limit: i32 = 5;
let workers: usize = 50;
sqlx::query!(
"INSERT INTO concurrency_counter(concurrency_id, job_uuids) VALUES ($1, '{}'::jsonb)",
key,
)
.execute(&db)
.await
.expect("seed empty counter row");
let mut tasks = Vec::with_capacity(workers);
for _ in 0..workers {
let db = db.clone();
backend/windmill-trigger-postgres/src/listener.rs
+18 -3
View File
@@ -136,7 +136,7 @@ impl PostgresSimpleClient {
/// Resolves the Postgres resource, validates that the configured publication and
/// replication slot still exist, and opens a fresh logical replication stream.
///
/// Returns `Error::BadConfig` when the publication or slot is missing, or the
/// Returns `Error::BadConfig` when the publication or slot is missing or invalidated, or the
/// data table is under roles (unrecoverable misconfigurations). Any other error is treated as transient
/// (connection refused, network interruption, ...) and is retried by the caller.
/// The resource is re-resolved on every call so credential rotations are picked
@@ -183,18 +183,33 @@ async fn connect_logical_replication_stream(
));
}
// `invalidation_reason` (PG 17+) and `conflicting` (PG 16) are read through jsonb so the
// query still runs on versions that lack those columns.
let replication_slot = client
.execute_query(&format!(
"SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}",
r#"SELECT COALESCE(
j->>'invalidation_reason',
CASE WHEN j->>'wal_status' = 'lost' THEN 'wal_removed'
WHEN j->>'conflicting' = 'true' THEN 'conflict with recovery' END
) FROM pg_replication_slots s, to_jsonb(s) j WHERE slot_name = {}"#,
quote_literal(replication_slot_name)
))
.await
.map_err(to_anyhow)?;
if !replication_slot.row_exist() {
let Some(slot) = replication_slot.iter().find_map(|m| match m {
SimpleQueryMessage::Row(row) => Some(row),
_ => None,
}) else {
return Err(Error::BadConfig(
ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(),
));
};
if let Some(reason) = slot.get(0) {
return Err(Error::BadConfig(format!(
"The replication slot `{replication_slot_name}` associated with this trigger has been invalidated by PostgreSQL (reason: `{reason}`). Changes since the invalidation cannot be replayed. Drop the slot (Advanced tab, slots) and save the trigger again to recreate it, or select another slot."
)));
}
client
backend/windmill-trigger/src/global_handler.rs
+1 -1
View File
@@ -342,7 +342,7 @@ pub async fn cancel_suspended_trigger_jobs(
let cancelled_jobs = cancel_jobs(
jobs_to_cancel,
&db,
authed.username.as_str(),
&authed,
w_id.as_str(),
true,
)
backend/windmill-types/src/flows.rs
+14
View File
@@ -377,6 +377,8 @@ impl FlowValue {
| Flow { .. }
| FlowScript { .. }
| Identity) => cb(&s, &module.id)?,
// Runs no script of its own.
AIDecision { .. } => {}
ForloopFlow { modules, .. } | WhileloopFlow { modules, .. } => {
Self::traverse_leafs(modules.iter().collect(), cb)?
}
@@ -1138,6 +1140,13 @@ pub enum FlowModuleValue {
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
tool_inputs: HashMap<String, HashMap<String, InputTransform>>,
},
/// One call to a decision model (such as TypeSafe's Jev) answering the typed `questions` of its
/// `input_transforms` about a `state`. A flow branches on the answers with a `BranchOne`.
AIDecision {
input_transforms: HashMap<String, InputTransform>,
#[serde(skip_serializing_if = "is_none_or_empty")]
tag: Option<String>,
},
}
fn is_none_or_empty(expr: &Option<String>) -> bool {
@@ -1282,6 +1291,10 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
agent: untagged.agent,
tool_inputs: untagged.tool_inputs.unwrap_or_default(),
}),
"aidecision" => Ok(FlowModuleValue::AIDecision {
input_transforms: untagged.input_transforms.unwrap_or_default(),
tag: untagged.tag,
}),
other => Err(serde::de::Error::unknown_variant(
other,
&[
@@ -1294,6 +1307,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue {
"rawscript",
"identity",
"aiagent",
"aidecision",
],
)),
}
backend/windmill-worker/src/ai/compaction.rs
+1
View File
@@ -651,6 +651,7 @@ async fn summarize_prefix(
.query_builder
.parse_streaming_response(resp, StreamEventProcessor::new_silent().boxed_sink())
.await?
.without_nul()
};
match parsed {
backend/windmill-worker/src/ai/tools.rs
+23 -4
View File
@@ -372,7 +372,8 @@ async fn enqueue_windmill_tool(
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::FlowScript { input_transforms, .. }
| FlowModuleValue::AIAgent { input_transforms, .. } => input_transforms,
| FlowModuleValue::AIAgent { input_transforms, .. }
| FlowModuleValue::AIDecision { input_transforms, .. } => input_transforms,
_ => {
return Err(Error::internal_err(format!(
"Unsupported tool: {}",
@@ -473,13 +474,15 @@ async fn enqueue_windmill_tool(
let has_nested_agent_tools = sub_tools.iter().any(|t| {
matches!(
t.value,
windmill_common::flows::ToolValue::FlowModule(FlowModuleValue::AIAgent { .. })
windmill_common::flows::ToolValue::FlowModule(
FlowModuleValue::AIAgent { .. } | FlowModuleValue::AIDecision { .. }
)
)
});
if has_nested_agent_tools {
return Err(Error::internal_err(
"AI agent tools cannot be nested beyond 2 levels. The nested agent tool contains \
AIAgent sub-tools, which would exceed the maximum nesting depth.".to_string()
AI agent or AI decision sub-tools, which would exceed the maximum nesting depth.".to_string()
));
}
let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id);
@@ -492,6 +495,18 @@ async fn enqueue_windmill_tool(
on_behalf_of: None,
}
}
// Runs as an AI agent job, whose handler finds this tool on the calling agent and answers
// it as a decision.
FlowModuleValue::AIDecision { tag, .. } => JobPayloadWithTag {
payload: JobPayload::AIAgent {
path: format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id),
},
tag: tag.filter(|t| !t.trim().is_empty()),
delete_after_use: tool_module.delete_after_use.unwrap_or(false),
delete_after_secs: None,
timeout: None,
on_behalf_of: None,
},
_ => {
return Err(Error::internal_err(format!(
"Unsupported tool: {}",
@@ -834,8 +849,12 @@ async fn execute_windmill_tools(
.result
.map(|value| value.0)
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
// A nested agent's or a decision's result is an envelope; the model sees `output`.
let is_agent = tool.module.as_ref().is_some_and(|module| {
matches!(module.get_value(), Ok(FlowModuleValue::AIAgent { .. }))
matches!(
module.get_value(),
Ok(FlowModuleValue::AIAgent { .. } | FlowModuleValue::AIDecision { .. })
)
});
let content = if is_agent && completed.success {
extract_ai_agent_output(&result).unwrap_or_else(|| result.get().to_string())
backend/windmill-worker/src/ai/utils.rs
+1
View File
@@ -720,6 +720,7 @@ pub fn any_tool_needs_previous_result(tools: &[Tool]) -> bool {
FlowModuleValue::RawScript { input_transforms, .. } => input_transforms,
FlowModuleValue::FlowScript { input_transforms, .. } => input_transforms,
FlowModuleValue::AIAgent { input_transforms, .. } => input_transforms,
FlowModuleValue::AIDecision { input_transforms, .. } => input_transforms,
_ => return false,
};
backend/windmill-worker/src/ai_executor.rs
+161 -8
View File
@@ -28,6 +28,7 @@ use windmill_mcp::McpClient;
use crate::ai::tools::McpClientStub as McpClient;
use windmill_ai::{
ai_providers::AIProvider,
decision::{decision_inputs, run_decision, AIDecisionArgs},
image_handler::upload_image_to_s3,
model_context::model_context_window,
providers::{
@@ -67,7 +68,10 @@ use crate::{
build_args_map, get_root_job_id, resolve_job_timeout, transform_json_value,
OccupancyMetrics, StreamNotifier,
},
handle_child::{run_future_with_polling_update_job_poller_graceful, GracefulPollOutcome},
handle_child::{
run_future_with_polling_update_job_poller,
run_future_with_polling_update_job_poller_graceful, GracefulPollOutcome,
},
};
lazy_static::lazy_static! {
@@ -81,6 +85,20 @@ lazy_static::lazy_static! {
"required": ["user_message"],
"additionalProperties": false,
}));
// Text only: the model writes the state as it reads the conversation, and a schema offering a
// union of types is refused by some providers' strict tool schemas.
static ref AI_DECISION_TOOL_SCHEMA: Box<RawValue> = to_raw_value(&serde_json::json!({
"type": "object",
"properties": {
"state": {
"type": "string",
"description": "What the questions are asked about, with only what they need.",
},
},
"required": ["state"],
"additionalProperties": false,
}));
}
const DEFAULT_MAX_AGENT_ITERATIONS: usize = 10;
@@ -498,7 +516,8 @@ fn overlay_tool_inputs(
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::FlowScript { input_transforms, .. }
| FlowModuleValue::AIAgent { input_transforms, .. } => input_transforms,
| FlowModuleValue::AIAgent { input_transforms, .. }
| FlowModuleValue::AIDecision { input_transforms, .. } => input_transforms,
_ => continue,
};
for (key, transform) in overrides {
@@ -713,6 +732,22 @@ pub async fn handle_ai_agent_job(
let summary = module.summary.clone();
let module_value = module.get_value()?;
if let FlowModuleValue::AIDecision { .. } = module_value {
return handle_ai_decision(
conn,
db,
job,
&local_args,
direct_parent_job_kind == JobKind::AIAgent,
canceled_by,
mem_peak,
occupancy_metrics,
worker_name,
)
.await;
}
let FlowModuleValue::AIAgent {
tools: module_tools,
omit_output_from_conversation,
@@ -720,7 +755,7 @@ pub async fn handle_ai_agent_job(
tool_inputs,
input_transforms: step_input_transforms,
..
} = module.get_value()?
} = module_value
else {
return Err(Error::internal_err(
"AI agent module is not an AI agent".to_string(),
@@ -820,16 +855,18 @@ pub async fn handle_ai_agent_job(
keep_authored_memory_id(&mut args, &step_input_transforms);
// Nesting is capped at flow → agent → nested agent. When this job is itself a nested tool,
// a linked resource's tool set may still contain AIAgent tools (the editor can't constrain a
// shared resource); don't advertise them — invoking one would only fail the depth check as a
// third-level agent.
// a linked resource's tool set may still contain AIAgent or AIDecision tools (the editor can't
// constrain a shared resource); don't advertise them — invoking one would only fail the depth
// check as a third level.
let tools = if direct_parent_job_kind == JobKind::AIAgent {
tools
.into_iter()
.filter(|t| {
!matches!(
&t.value,
ToolValue::FlowModule(FlowModuleValue::AIAgent { .. })
ToolValue::FlowModule(
FlowModuleValue::AIAgent { .. } | FlowModuleValue::AIDecision { .. }
)
)
})
.collect()
@@ -1004,6 +1041,18 @@ pub async fn handle_ai_agent_job(
None,
)
}
FlowModuleValue::AIDecision { input_transforms, .. } => {
// The calling model supplies only the state: the questions are the tool.
let description = decision_tool_description(&input_transforms);
(
Some(
RawValue::from_string(AI_DECISION_TOOL_SCHEMA.get().to_string())
.expect("AI_DECISION_TOOL_SCHEMA should always be valid JSON"),
),
input_transforms,
description,
)
}
_ => {
return Err(Error::internal_err(format!(
"Unsupported tool: {}",
@@ -1242,6 +1291,12 @@ pub async fn run_agent(
job_completed_tx: crate::JobCompletedSender,
) -> error::Result<Box<RawValue>> {
let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text);
if args.provider.kind.is_decision_provider() {
return Err(Error::BadRequest(format!(
"{:?} serves decision models, which answer typed questions rather than messages: use an AI decision step",
args.provider.kind
)));
}
let credentials = args.provider.to_provider_credentials(db).await?;
let base_url = &credentials.base_url;
let api_key = credentials.api_key.as_deref().unwrap_or("");
@@ -1860,7 +1915,7 @@ pub async fn run_agent(
query_builder.parse_image_response(resp).await?
}
};
Ok(parsed)
Ok(parsed.without_nul())
}.await;
match attempt {
@@ -2379,6 +2434,104 @@ pub async fn run_agent(
}))
}
/// An AI decision, run as a flow step or as a tool of an agent: its questions answered about its
/// state in one call, with no loop, tools, memory or stream. It writes no conversation row: it is
/// not an agent step, so a chat flow posts its result like any other step's.
async fn handle_ai_decision(
conn: &Connection,
db: &DB,
job: &MiniPulledJob,
local_args: &HashMap<String, Box<RawValue>>,
as_tool: bool,
canceled_by: &mut Option<CanceledBy>,
mem_peak: &mut i32,
occupancy_metrics: &mut OccupancyMetrics,
worker_name: &str,
) -> error::Result<Box<RawValue>> {
let args = serde_json::from_str::<AIDecisionArgs>(&serde_json::to_string(local_args)?)?;
if !args.provider.kind.runs_decisions() {
return Err(Error::BadRequest(format!(
"An AI decision runs on a TypeSafe, Cloudflare or OpenAI resource, not {:?}",
args.provider.kind
)));
}
let (state, questions) = decision_inputs(args.state.as_ref(), args.questions.as_ref())?;
let credentials = args.provider.to_provider_credentials(db).await?;
let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout)
.await
.0;
// Under the job poller, so the job keeps its heartbeat and a cancel drops the request.
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
conn,
mem_peak,
canceled_by,
run_decision(
&credentials,
args.provider.get_model(),
state,
questions,
timeout,
),
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
windmill_common::feature_usage::log_feature_usage(
"ai_decision",
"run",
if as_tool { "tool" } else { "step" },
);
Ok(to_raw_value(&result))
}
/// What a decision tool tells the calling model it answers, read off its static questions. None
/// when they come from an expression, which only a run evaluates.
fn decision_tool_description(input_transforms: &HashMap<String, InputTransform>) -> Option<String> {
let InputTransform::Static { value } = input_transforms.get("questions")? else {
return None;
};
let questions = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(value.get())
.ok()
.filter(|questions| !questions.is_empty())?;
let described = questions
.iter()
.map(|(name, question)| {
let criteria = question.get("criteria");
let kind = match question.get("type").and_then(|t| t.as_str()) {
Some("choice") => {
let options = criteria
.and_then(|c| c.as_object())
.map(|c| c.keys().cloned().collect::<Vec<_>>().join(", "))
.unwrap_or_default();
format!("one of {options}")
}
Some("score") => {
let levels = criteria
.and_then(|c| c.as_array())
.map(|c| {
c.iter()
.filter_map(|l| l.as_str())
.collect::<Vec<_>>()
.join(" < ")
})
.unwrap_or_default();
format!("a score over {levels}")
}
_ => "yes or no".to_string(),
};
format!("{name} ({kind})")
})
.collect::<Vec<_>>()
.join("; ");
Some(format!(
"Answers these questions about the state it is given, with probabilities: {described}."
))
}
/// Whether the step asked for its answer as it is generated. Absence means on, matching the
/// schema's own default: a step that never wrote the key never had an opinion, and an answer
/// arriving as it is written is what people expect. Only an explicit `false` holds it back.
backend/windmill-worker/src/common.rs
+167 -23
View File
@@ -1068,12 +1068,87 @@ pub async fn start_child_process(
Ok(child)
}
/// Which limit a resolved job timeout comes from.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TimeoutSource {
/// The timeout set on the script or the flow step.
Custom,
/// What is left of a wall clock that the phases of one job spend down together.
RemainingBudget,
/// The instance `job_default_timeout` setting.
InstanceDefault,
/// The instance ceiling (`TIMEOUT`), which no other timeout can exceed.
InstanceMax,
}
impl TimeoutSource {
/// How to name a limit of `secs` seconds in a message shown to the job's author.
pub fn describe(&self, secs: u64) -> String {
match self {
TimeoutSource::Custom => {
format!("the {secs}s custom timeout set on the script or flow step")
}
TimeoutSource::RemainingBudget => format!(
"the {secs}s left of the job's timeout, which its phases share"
),
TimeoutSource::InstanceDefault => format!(
"the {secs}s instance default timeout (instance setting 'Default timeout'); set a longer custom timeout on the script or flow step to raise it"
),
TimeoutSource::InstanceMax => format!(
"the {secs}s instance maximum job duration (TIMEOUT env variable of the worker), which no custom or default timeout can exceed"
),
}
}
}
lazy_static! {
static ref RUNNING_JOB_CUSTOM_TIMEOUTS: std::sync::Mutex<HashMap<Uuid, Option<i32>>> =
std::sync::Mutex::new(HashMap::new());
}
/// Makes a running job's custom timeout visible to the phases that are not handed it (dependency
/// resolution, install, build), for as long as the guard lives.
pub struct RunningJobCustomTimeout(Uuid);
impl RunningJobCustomTimeout {
pub fn register(job_id: Uuid, custom_timeout_secs: Option<i32>) -> Self {
if let Ok(mut m) = RUNNING_JOB_CUSTOM_TIMEOUTS.lock() {
m.insert(
job_id,
windmill_common::runnable_settings::none_if_non_positive(custom_timeout_secs),
);
}
Self(job_id)
}
}
impl Drop for RunningJobCustomTimeout {
fn drop(&mut self) {
if let Ok(mut m) = RUNNING_JOB_CUSTOM_TIMEOUTS.lock() {
m.remove(&self.0);
}
}
}
/// `None` when the job is not registered, `Some(None)` when it runs without a custom timeout.
fn registered_custom_timeout(job_id: Uuid) -> Option<Option<i32>> {
RUNNING_JOB_CUSTOM_TIMEOUTS
.lock()
.ok()
.and_then(|m| m.get(&job_id).copied())
}
/// Resolves the time limit of one phase of a job and where that limit comes from.
///
/// A phase called without a custom timeout runs under the instance default. A job's custom
/// timeout may only lengthen such a phase, never shorten it: a short run timeout must not kill
/// a dependency install that the instance default allows.
pub async fn resolve_job_timeout(
_conn: &Connection,
_w_id: &str,
_job_id: Uuid,
job_id: Uuid,
custom_timeout_secs: Option<i32>,
) -> (Duration, Option<String>, bool) {
) -> (Duration, Option<String>, TimeoutSource) {
let mut warn_msg: Option<String> = None;
#[cfg(feature = "cloud")]
let cloud_premium_workspace =
@@ -1091,6 +1166,7 @@ pub async fn resolve_job_timeout(
let global_max_timeout_duration =
Duration::from_secs(max_job_duration_secs(cloud_premium_workspace));
let max_secs = global_max_timeout_duration.as_secs();
// A `custom_timeout_secs <= 0` is not a 0-second limit but "unset": fall through to the
// default/global-max timeout instead of killing the job immediately.
@@ -1098,34 +1174,95 @@ pub async fn resolve_job_timeout(
Some(timeout_secs)
if Duration::from_secs(timeout_secs as u64) < global_max_timeout_duration =>
{
(Duration::from_secs(timeout_secs as u64), warn_msg, true)
// A phase handed anything but the job's own custom timeout was handed the rest of a
// shared wall clock, whatever limit that clock was started from.
let source =
if registered_custom_timeout(job_id).is_none_or(|t| t == Some(timeout_secs)) {
TimeoutSource::Custom
} else {
TimeoutSource::RemainingBudget
};
(Duration::from_secs(timeout_secs as u64), warn_msg, source)
}
Some(timeout_secs) => {
warn_msg = Some(format!("WARNING: Custom job timeout of {timeout_secs} seconds was greater than the maximum timeout. It will be ignored and the max timeout will be used instead"));
warn_msg = Some(format!("WARNING: Custom job timeout of {timeout_secs} seconds was greater than the instance maximum job duration of {max_secs} seconds (TIMEOUT env variable of the worker). It will be ignored and the max timeout will be used instead"));
tracing::warn!(warn_msg);
(global_max_timeout_duration, warn_msg, false)
(
global_max_timeout_duration,
warn_msg,
TimeoutSource::InstanceMax,
)
}
None => {
// fallback to default timeout or max if not set
let default_timeout = match JOB_DEFAULT_TIMEOUT.read().await.clone() {
None => global_max_timeout_duration,
let (default_timeout, source) = match JOB_DEFAULT_TIMEOUT.read().await.clone() {
None => (global_max_timeout_duration, TimeoutSource::InstanceMax),
Some(default_timeout_secs)
if Duration::from_secs(default_timeout_secs as u64)
< global_max_timeout_duration =>
{
Duration::from_secs(default_timeout_secs as u64)
(
Duration::from_secs(default_timeout_secs as u64),
TimeoutSource::InstanceDefault,
)
}
Some(default_timeout_secs) => {
warn_msg = Some(format!("WARNING: Default job timeout of {default_timeout_secs} seconds was greater than the maximum timeout. It will be ignored and the global max timeout will be used instead"));
warn_msg = Some(format!("WARNING: Default job timeout of {default_timeout_secs} seconds was greater than the instance maximum job duration of {max_secs} seconds (TIMEOUT env variable of the worker). It will be ignored and the max timeout will be used instead"));
tracing::warn!(warn_msg);
global_max_timeout_duration
(global_max_timeout_duration, TimeoutSource::InstanceMax)
}
};
(default_timeout, warn_msg, false)
let job_custom_timeout = registered_custom_timeout(job_id)
.flatten()
.map(|secs| Duration::from_secs(secs as u64));
let (timeout, source) = lengthen_with_job_custom_timeout(
default_timeout,
source,
job_custom_timeout,
global_max_timeout_duration,
);
(timeout, warn_msg, source)
}
}
}
fn lengthen_with_job_custom_timeout(
phase_timeout: Duration,
phase_source: TimeoutSource,
job_custom_timeout: Option<Duration>,
max_timeout: Duration,
) -> (Duration, TimeoutSource) {
match job_custom_timeout {
Some(custom) if custom > phase_timeout && custom < max_timeout => {
(custom, TimeoutSource::Custom)
}
Some(custom) if custom > phase_timeout => (max_timeout, TimeoutSource::InstanceMax),
_ => (phase_timeout, phase_source),
}
}
#[cfg(test)]
mod job_timeout_tests {
use super::{lengthen_with_job_custom_timeout, TimeoutSource};
use std::time::Duration;
// A phase that is not handed the job's custom timeout (dependency install, build) runs under
// the instance default. The custom timeout may extend it up to the instance max, but a
// shorter one must leave it alone.
#[test]
fn job_custom_timeout_only_lengthens_a_setup_phase() {
let s = Duration::from_secs;
let default = (s(120), TimeoutSource::InstanceDefault);
let resolve = |custom: Option<u64>| {
lengthen_with_job_custom_timeout(default.0, default.1, custom.map(s), s(600))
};
assert_eq!(resolve(None), default);
assert_eq!(resolve(Some(240)), (s(240), TimeoutSource::Custom));
assert_eq!(resolve(Some(30)), default);
assert_eq!(resolve(Some(900)), (s(600), TimeoutSource::InstanceMax));
}
}
/// Compute the nsjail timeout (in seconds) with a 15s buffer so handle_child fires first.
pub async fn resolve_nsjail_timeout(
conn: &Connection,
@@ -1517,6 +1654,10 @@ async fn hash_args(
#[allow(unused)] ignore_s3_path: bool,
) {
if let Some(Json(hm)) = v {
// Resolving an AWS OIDC storage mints a token for the job, so each storage is
// resolved once however many arguments point into it.
#[cfg(feature = "parquet")]
let mut storages: HashMap<Option<String>, Option<ObjectStoreResource>> = HashMap::new();
for k in hm.keys().sorted() {
hasher.update(k.as_bytes());
let arg_value = hm.get(k).unwrap();
@@ -1524,18 +1665,21 @@ async fn hash_args(
#[cfg(feature = "parquet")]
let etag = match serde_json::from_str::<S3Object>(arg_value.get()).ok() {
Some(s3_object) => {
let s3_resource = get_workspace_s3_resource_path(
db,
client,
workspace_id,
s3_object.storage.as_ref(),
job_id,
)
.await
.ok()
.flatten();
match s3_resource {
Some(s3_resource) => get_etag_or_empty(&s3_resource, s3_object).await,
if !storages.contains_key(&s3_object.storage) {
let s3_resource = get_workspace_s3_resource_path(
db,
client,
workspace_id,
s3_object.storage.as_ref(),
job_id,
)
.await
.ok()
.flatten();
storages.insert(s3_object.storage.clone(), s3_resource);
}
match storages.get(&s3_object.storage).and_then(Option::as_ref) {
Some(s3_resource) => get_etag_or_empty(s3_resource, s3_object).await,
None => None,
}
}
backend/windmill-worker/src/dbt_executor.rs
+5 -5
View File
@@ -779,6 +779,7 @@ pub(crate) async fn dbt_dep(
// A dependency job carries no per-job timeout of its own, but its phases
// still share one wall clock rather than each getting the instance-wide
// one.
let _custom_timeout = crate::common::RunningJobCustomTimeout::register(*job_id, None);
let deadline = JobDeadline::start(&conn, w_id, *job_id, None).await;
let mut ctx = JobCtx { mem_peak, canceled_by, occupancy_metrics, worker_name, deadline };
let prepared = prepare_project(
@@ -3560,11 +3561,10 @@ async fn attach_column_index(
.iter()
.map(|n| n.unique_id.as_str())
.collect();
let index =
crate::dbt_column_index::collect(
p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept,
)
.await?;
let index = crate::dbt_column_index::collect(
p, descriptor, inv, command, ctx, job_id, w_id, conn, &kept,
)
.await?;
drop(kept);
let Some(index) = index else {
return Ok(());
backend/windmill-worker/src/handle_child.rs
+18 -17
View File
@@ -49,7 +49,7 @@ use futures::{
stream, StreamExt,
};
use crate::common::{resolve_job_timeout, OccupancyMetrics, StreamNotifier};
use crate::common::{resolve_job_timeout, OccupancyMetrics, StreamNotifier, TimeoutSource};
use crate::job_logger::{append_job_logs, append_result_stream, append_with_limit, strip_nul};
use crate::job_logger_oss::process_streaming_log_lines;
use crate::worker_utils::{ping_job_status, update_worker_ping_from_job};
@@ -181,7 +181,7 @@ pub async fn handle_child(
enum KillReason {
TooManyLogs,
Timeout { is_job_specific: bool },
Timeout { phase: String, source: TimeoutSource, secs: u64 },
Cancelled(Option<CanceledBy>),
AlreadyCompleted,
}
@@ -190,11 +190,9 @@ pub async fn handle_child(
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
KillReason::TooManyLogs => f.write_str("too many logs (max size: 2MB)"),
KillReason::Timeout { is_job_specific } => f.write_str(if *is_job_specific {
"timeout after exceeding job-specific duration limit"
} else {
"timeout after exceeding instance-wide job duration limit"
}),
KillReason::Timeout { phase, source, secs } => {
write!(f, "timeout: `{phase}` exceeded {}", source.describe(*secs))
}
KillReason::Cancelled(canceled_by) => {
let mut reason = "cancelled".to_string();
if let Some(canceled_by) = canceled_by {
@@ -212,7 +210,7 @@ pub async fn handle_child(
}
}
let (timeout_duration, timeout_warn_msg, is_job_specific) =
let (timeout_duration, timeout_warn_msg, timeout_source) =
resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
append_logs(&job_id, w_id, msg.as_str(), conn).await;
@@ -224,7 +222,11 @@ pub async fn handle_child(
biased;
result = Box::into_pin(child.wait()) => return result.map(Ok),
Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
_ = sleep(timeout_duration) => KillReason::Timeout { is_job_specific },
_ = sleep(timeout_duration) => KillReason::Timeout {
phase: child_name.to_string(),
source: timeout_source,
secs: timeout_duration.as_secs(),
},
ex = update_job, if job_id != Uuid::nil() => match ex {
UpdateJobPollingExit::Done(canceled_by) => KillReason::Cancelled(canceled_by),
UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
@@ -233,17 +235,16 @@ pub async fn handle_child(
tx.send(()).expect("rx should never be dropped");
drop(tx);
let timeout_reason = format!(
"`{child_name}` exceeded {}",
timeout_source.describe(timeout_duration.as_secs())
);
let set_reason = async {
if matches!(kill_reason, KillReason::Timeout { .. }) {
match conn {
Connection::Sql(db) => {
if let Err(err) = set_job_cancelled_query(
job_id,
db,
"timeout",
&format!("duration > {}", timeout_duration.as_secs()),
)
.await
if let Err(err) =
set_job_cancelled_query(job_id, db, "timeout", &timeout_reason).await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
}
@@ -255,7 +256,7 @@ pub async fn handle_child(
None,
&JobCancelled {
canceled_by: "timeout".to_string(),
reason: format!("duration > {}", timeout_duration.as_secs()),
reason: timeout_reason.clone(),
},
)
.await
backend/windmill-worker/src/snowflake_executor.rs
+1 -1
View File
@@ -855,7 +855,7 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value {
}
})
}
"binary" => {
"binary" | "varbinary" => {
// convert base64 to hex as expected by snowflake
let bytes = engine::general_purpose::STANDARD
.decode(arg_v.as_str().unwrap_or(""))
backend/windmill-worker/src/sql_s3_input.rs
+1 -1
View File
@@ -56,7 +56,7 @@ pub async fn fetch_s3object_as_json_text(
crate::worker_utils::JobPingHeartbeat::start(conn, job_id, "s3object materialization");
let bytes = client
.download_s3_file(workspace_id, &obj.s3, obj.storage.clone())
.download_s3_file(workspace_id, &obj.s3, obj.storage.clone(), None)
.await
.with_context(|| format!("Failed to download S3 object `{}`", obj.s3))?;
backend/windmill-worker/src/worker.rs
+1
View File
@@ -5401,6 +5401,7 @@ async fn handle_code_execution_job(
precomputed_agent_info: Option<PrecomputedAgentInfo>,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
let _custom_timeout = crate::common::RunningJobCustomTimeout::register(job.id, job.timeout);
let script_hash = || {
job.runnable_id
.ok_or_else(|| Error::internal_err("expected script hash"))
backend/windmill-worker/src/worker_flow.rs
+39 -4
View File
@@ -192,6 +192,7 @@ pub async fn update_flow_status_after_job_completion(
flow_job_duration,
stop_early_override,
has_triggered_error_handler: false,
recovered_by_child: false,
};
let mut step_failure = step_failure;
loop {
@@ -211,6 +212,7 @@ pub async fn update_flow_status_after_job_completion(
worker_dir,
rec.stop_early_override,
rec.has_triggered_error_handler,
rec.recovered_by_child,
worker_name,
job_completed_tx.clone(),
flow_runners.clone(),
@@ -240,6 +242,7 @@ pub async fn update_flow_status_after_job_completion(
worker_dir,
rec.stop_early_override,
rec.has_triggered_error_handler,
false,
worker_name,
job_completed_tx.clone(),
flow_runners.clone(),
@@ -293,6 +296,10 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
flow_job_duration: Option<FlowJobDuration>,
stop_early_override: Option<bool>,
has_triggered_error_handler: bool,
/// The failed child's error handler returned `recover: true`. The parent still handles the
/// failure as it would any other, so the same steps run; only a root flow that this failure
/// ends turns green.
recovered_by_child: bool,
}
#[derive(Deserialize)]
@@ -427,6 +434,7 @@ pub async fn update_flow_status_after_job_completion_internal(
worker_dir: &str,
stop_early_override: Option<bool>,
has_triggered_error_handler: bool,
mut recovered_by_child: bool,
worker_name: &str,
job_completed_tx: JobCompletedSender,
flow_runners: Option<Arc<FlowRunners>>,
@@ -449,6 +457,7 @@ pub async fn update_flow_status_after_job_completion_internal(
skip_if_stop_early,
nresult,
is_failure_step,
failure_step_recovers,
_cleanup_module,
chat_ai_info,
) = {
@@ -1096,6 +1105,9 @@ pub async fn update_flow_status_after_job_completion_internal(
} else {
has_triggered_error_handler = false;
}
// Recovered iterations are stored as successes, so a failed parallel
// node failed on an iteration that did not recover.
recovered_by_child = false;
(success, Some(new_status))
} else {
add_time!(bench, "handle parallel flow start");
@@ -1823,6 +1835,13 @@ pub async fn update_flow_status_after_job_completion_internal(
_ => false,
};
let failure_step_recovers = is_failure_step && result_has_recover_true(nresult.clone());
// An error this flow raises on its own (a stop predicate's error message) is a new
// failure that no error handler recovered.
if stop_early_err_msg.is_some() {
recovered_by_child = false;
}
let chat_ai_info = ChatAiInfo {
chat_input_enabled: old_status.chat_input_enabled.unwrap_or(false),
conversation_id: old_status.memory_id,
@@ -1837,6 +1856,7 @@ pub async fn update_flow_status_after_job_completion_internal(
skip_if_stop_early,
nresult,
is_failure_step,
failure_step_recovers,
old_status.cleanup_module,
chat_ai_info,
)
@@ -1844,6 +1864,10 @@ pub async fn update_flow_status_after_job_completion_internal(
let flow_job = Arc::new(flow_job);
// An inner flow that ends on a recovered failure stays failed, as parents read their
// children's stored status (a parallel node counts its iterations' statuses), so only the
// root flow turns green.
let ends_root = !(flow_job.is_flow_step() && flow_job.parent_job.is_some());
let done = if !should_continue_flow {
{
let logs = if flow_job.is_canceled() {
@@ -1851,7 +1875,9 @@ pub async fn update_flow_status_after_job_completion_internal(
} else if stop_early {
format!("Flow job stopped early because of a stop early predicate returning true\n")
} else if is_failure_step {
format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, result_has_recover_true(nresult.clone()))
format!("Flow job completed with error, and error handler was triggered.\nIt completed with {}, and with recover: {}\n", if success { "success" } else { "error" }, failure_step_recovers)
} else if !success && recovered_by_child && ends_root {
format!("Flow job completed with success: the step that failed had its error recovered by the error handler (recover: true)\n")
} else {
format!(
"Flow job completed with {}\n",
@@ -2048,7 +2074,11 @@ pub async fn update_flow_status_after_job_completion_internal(
.await;
}
let success = success && (!is_failure_step || result_has_recover_true(nresult.clone()));
let success = if is_failure_step {
success && failure_step_recovers
} else {
success || (recovered_by_child && ends_root)
};
add_time!(bench, "flow status update 1");
@@ -2149,6 +2179,8 @@ pub async fn update_flow_status_after_job_completion_internal(
.await;
// override prior child's success so the parent learns this flow failed
success = false;
// a new failure, which no error handler recovered
recovered_by_child = false;
true
}
Ok(_) => false,
@@ -2178,6 +2210,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None
},
has_triggered_error_handler: has_triggered_error_handler || is_failure_step,
recovered_by_child: failure_step_recovers || (recovered_by_child && !success),
},
));
}
@@ -4184,7 +4217,8 @@ async fn push_next_flow_job(
| FlowModuleValue::RawScript { input_transforms, .. }
| FlowModuleValue::FlowScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. }
| FlowModuleValue::AIAgent { input_transforms, .. },
| FlowModuleValue::AIAgent { input_transforms, .. }
| FlowModuleValue::AIDecision { input_transforms, .. },
) => {
let ctx = get_transform_context(&flow_job, &previous_id, &status);
transform_context = Some(ctx);
@@ -5488,7 +5522,8 @@ async fn compute_next_flow_transform(
NextStatus::NextStep,
))
}
FlowModuleValue::AIAgent { tag, .. } => {
// A decision runs as an AI agent job, whose handler answers it as a decision.
FlowModuleValue::AIAgent { tag, .. } | FlowModuleValue::AIDecision { tag, .. } => {
let path = get_path(flow_job, status, module);
let payload = JobPayload::AIAgent { path };
Ok(NextFlowTransform::Continue(
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.823.1";
export const VERSION = "v1.826.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
chat-sdk/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-chat",
"version": "1.823.1",
"version": "1.826.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-chat",
"version": "1.823.1",
"version": "1.826.0",
"license": "Apache-2.0",
"devDependencies": {
"@ai-sdk/react": "^4.0.102",
chat-sdk/package.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "windmill-chat",
"description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app",
"version": "1.823.1",
"version": "1.826.0",
"author": "Ruben Fiszel",
"license": "Apache-2.0",
"homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme",
cli/src/commands/app/new.ts
+1 -1
View File
@@ -908,7 +908,7 @@ const command = new Command()
)
.option(
"--schema <schema:string>",
"Schema to use with --datatable. Created (CREATE SCHEMA IF NOT EXISTS) if it doesn't already exist."
"Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it."
)
.option(
"--overwrite",
cli/src/core/constants.ts
+1 -1
View File
@@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork";
// (e.g. utils.ts) can read it without importing main.ts and creating a circular
// dependency (main → workspace → utils → main) that triggers a TDZ.
// Re-exported from main.ts for backwards compatibility.
export const VERSION = "1.823.1";
export const VERSION = "1.826.0";
cli/src/guidance/core.ts
+1 -1
View File
@@ -90,7 +90,7 @@ A data pipeline is NOT a flow: it is a set of independent scripts in one folder,
## Raw App Development
You MUST use the \`raw-app\` skill to create or modify raw apps.
When a new app needs to be created, YOU run \`wmill app new\` yourself with \`--summary\`, \`--path\`, and \`--framework\` flags (and any other relevant flags). Do NOT ask the user to run it. If you don't have the values for those flags, ask the user via \`AskUserQuestion\` (one call, all missing fields) — never invent them. See the \`raw-app\` skill for the full procedure.
When a new app needs to be created, YOU run \`wmill app new\` yourself with \`--summary\`, \`--path\`, and \`--framework\` flags (and any other relevant flags). Do NOT ask the user to run it. If you don't have a summary or a path, ask the user via \`AskUserQuestion\` (one call, all missing fields) — never invent them. The framework is not asked: \`react19\` unless the user or the project's existing apps say otherwise. An app that stores data also needs its data table and schema settled with the user first. See the \`raw-app\` skill for the full procedure.
## Triggers
cli/src/guidance/skills.gen.ts
Generated
+149 -17
View File
File diff suppressed because one or more lines are too long
docker/DockerfileExtra
+3 -3
View File
@@ -44,9 +44,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
RUN set -eux; \
arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
case "$arch" in \
'amd64') targz='go1.26.0.linux-amd64.tar.gz' ;; \
'arm64') targz='go1.26.0.linux-arm64.tar.gz' ;; \
'armhf') targz='go1.26.0.linux-armv6l.tar.gz' ;; \
'amd64') targz='go1.27.1.linux-amd64.tar.gz' ;; \
'arm64') targz='go1.27.1.linux-arm64.tar.gz' ;; \
'armhf') targz='go1.27.1.linux-armv6l.tar.gz' ;; \
*) echo >&2 "error: unsupported architecture '$arch'"; exit 1 ;; \
esac; \
wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz"
docker/entrypoint-extra.sh
+1 -1
View File
@@ -134,7 +134,7 @@ fi
# Setup Monaco temp directory for LSP
mkdir -p /tmp/monaco
if [ ! -f /tmp/monaco/go.mod ]; then
echo -e "module mymod\ngo 1.26" > /tmp/monaco/go.mod
echo -e "module mymod\ngo 1.27" > /tmp/monaco/go.mod
fi
echo "[entrypoint] Starting Windmill Extra Services"

Some files were not shown because too many files have changed in this diff Show More