diff --git a/.agents/skills/local-review-codex/SKILL.md b/.agents/skills/local-review-codex/SKILL.md
new file mode 100644
index 0000000000..cc932f7a4b
--- /dev/null
+++ b/.agents/skills/local-review-codex/SKILL.md
@@ -0,0 +1,50 @@
+---
+name: local-review-codex
+description: Run the CI Codex PR review locally against this branch's unpushed work (committed + uncommitted) before pushing. Same policy, model, and reasoning effort as the codex-pr-review GitHub action.
+---
+
+# Local Codex Review (pre-push)
+
+Runs the exact same review Codex performs in CI (`.github/workflows/codex-pr-review.yml`),
+but locally and scoped to work you have not pushed yet — so you catch what CI would flag
+before the PR exists. Use this before `git push` on a non-trivial change.
+
+**Correspondence with CI** — identical:
+- Policy: `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test coverage).
+- Model: `gpt-5.6-sol`, `model_reasoning_effort="xhigh"`.
+- Output: markdown starting with `## Codex Review`, findings tagged P0 / P1 / P2 with file:line.
+
+**Differences from CI** — local-only:
+- Scope is the current branch vs `main` at the merge-base, **including uncommitted changes** (CI reviews a pushed PR diff).
+- Sandbox is `read-only` (CI uses `danger-full-access` on an ephemeral runner). Codex reads the diff and files but cannot modify your working tree.
+- Fresh context is inherent: `codex exec` is a separate cold process, so it does not anchor on the current chat session — the same reason `local-review` insists on a subagent.
+
+## Prerequisites
+
+- `codex` CLI **>= 0.144.1** installed and authed (`codex login` or `OPENAI_API_KEY`). Older CLIs reject `gpt-5.6-sol` with "requires a newer version of Codex". Upgrade with `npm install --global @openai/codex@0.144.1` (may need `sudo` for a global install). Keep this in sync with the pin in `.github/workflows/codex-pr-review.yml`.
+- `git fetch` the base ref if it's stale, so the merge-base is accurate.
+
+## Run
+
+```bash
+bash .agents/skills/local-review-codex/run.sh # review vs main (default)
+bash .agents/skills/local-review-codex/run.sh # review vs a different base ref
+```
+
+Invoke with `bash` (or run the executable directly) — the script needs Bash for
+`set -o pipefail`; `sh` is Dash on Debian/Ubuntu and would fail. If `main` isn't a
+local branch (e.g. a fresh single-branch checkout), the runner falls back to
+`origin/main` automatically.
+
+The script computes `BASE_SHA = git merge-base HEAD `, feeds Codex `REVIEW.md` plus a
+diff context pointing at `git diff ` (which folds in uncommitted edits), and prints
+the review. It writes only temp files — nothing lands in the working tree.
+
+## Relaying the result
+
+Print the Codex output verbatim. Do not re-summarize or filter it — the value of a cold Codex
+pass is surfacing what the current session would rationalize away. Then decide with the user
+whether to address findings before pushing.
+
+For a Claude-native review instead, use `local-review` (branch-diff-reviewer subagent). This
+skill is the Codex counterpart; run both for independent perspectives.
diff --git a/.agents/skills/local-review-codex/run.sh b/.agents/skills/local-review-codex/run.sh
new file mode 100755
index 0000000000..d6491099c2
--- /dev/null
+++ b/.agents/skills/local-review-codex/run.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Local Codex review — mirrors the .github/workflows/codex-pr-review.yml CI job,
+# but scoped to this branch's unpushed work (committed + uncommitted) so you can
+# review before pushing. Same policy (REVIEW.md), same model (gpt-5.6-sol) and
+# reasoning effort (xhigh) as CI. Runs read-only: Codex cannot modify your tree.
+#
+# Usage: run.sh [BASE_REF] (BASE_REF defaults to "main")
+set -euo pipefail
+
+BASE_REF="${1:-main}"
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+cd "$REPO_ROOT"
+
+if ! command -v codex >/dev/null 2>&1; then
+ echo "codex CLI not found. Install with: npm install --global @openai/codex@0.144.1" >&2
+ exit 1
+fi
+
+# Resolve the base to a concrete commit, preferring a local ref but falling back to
+# the remote-tracking ref — checkouts (CI, single-branch clones) often have only
+# origin/main, not a local main.
+if git rev-parse --verify --quiet "${BASE_REF}^{commit}" >/dev/null; then
+ BASE_COMMITISH="$BASE_REF"
+elif git rev-parse --verify --quiet "origin/${BASE_REF}^{commit}" >/dev/null; then
+ BASE_COMMITISH="origin/${BASE_REF}"
+else
+ echo "Base ref '$BASE_REF' not found as '$BASE_REF' or 'origin/$BASE_REF'. Try: git fetch origin $BASE_REF" >&2
+ exit 1
+fi
+
+# Diff from the merge-base so only this branch's changes are reviewed. Using the
+# base SHA with a single-ref `git diff` also folds in uncommitted working-tree edits,
+# but `git diff` never sees untracked files — those are gathered separately below so
+# brand-new files (a whole new module, a new skill dir) are not silently skipped.
+BASE_SHA="$(git merge-base HEAD "$BASE_COMMITISH")"
+HEAD_SHA="$(git rev-parse HEAD)"
+UNTRACKED="$(git ls-files --others --exclude-standard)"
+
+if [ "$BASE_SHA" = "$HEAD_SHA" ] && git diff --quiet "$BASE_SHA" && [ -z "$UNTRACKED" ]; then
+ echo "No changes vs $BASE_REF — nothing to review." >&2
+ exit 0
+fi
+
+PROMPT="$(mktemp)"
+OUT="$(mktemp)"
+trap 'rm -f "$PROMPT" "$OUT"' EXIT
+
+# REVIEW.md is the shared policy CI feeds Codex. Append the local output-format
+# and diff context inline (CI reads these from a generated context file; inlining
+# keeps the working tree clean — no scratch files land in the repo).
+cat REVIEW.md > "$PROMPT"
+cat >> "$PROMPT" < no .pdb and no LNK1318 type-server limit).
+ CARGO_PROFILE_DEV_DEBUG: "0"
+ CARGO_PROFILE_TEST_DEBUG: "0"
# Tests' poll-time stack frames (deep nested async fn chains in
# debug builds) reach ~1.8MB. 4MB gives ~2x headroom against flaky
# overflows under parallel-test contention.
diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml
index 0be727d4bd..3d6d7d8dee 100644
--- a/.github/workflows/backend-test.yml
+++ b/.github/workflows/backend-test.yml
@@ -90,7 +90,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
- toolchain: 1.93.0
+ toolchain: 1.97.0
- name: Fix stale v8 build cache
working-directory: ./backend
run: |
@@ -246,6 +246,18 @@ jobs:
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
CARGO_BUILD_JOBS: 12
+ # backend/Cargo.toml leaves profile.dev at the default debug = 2 for
+ # the (large) windmill workspace crates; that debug info is emitted
+ # into every object file and embedded in each test binary. Across the
+ # full --all --features build it is the dominant memory/disk consumer
+ # when mold links the windmill-api-integration-tests binary, tipping
+ # the runner over (lost runner reported as a canceled step). CI needs
+ # no debug info, so drop it entirely for the dev/test profiles here.
+ # (test profile inherits dev, but the workspace crates link in as
+ # dev-profile deps, so both must be set.) CI-only; local dev builds
+ # are unaffected.
+ CARGO_PROFILE_DEV_DEBUG: "0"
+ CARGO_PROFILE_TEST_DEBUG: "0"
# Tests' poll-time stack frames (deep nested async fn chains in
# debug builds) reach ~1.8MB, leaving very thin headroom on the
# default 2MB thread stack. 4MB gives ~2x buffer against flaky
diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml
index 7fe2d4e416..2d573c6959 100644
--- a/.github/workflows/build_windows_worker_.yml
+++ b/.github/workflows/build_windows_worker_.yml
@@ -33,7 +33,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
- toolchain: 1.93.0
+ toolchain: 1.97.0
- name: Substitute EE code
shell: bash
diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml
index ea622914bc..68bea1d715 100644
--- a/.github/workflows/codex-pr-review.yml
+++ b/.github/workflows/codex-pr-review.yml
@@ -104,24 +104,36 @@ jobs:
IS_FORK="$EVENT_FORK"
PR_AUTHOR="$EVENT_AUTHOR"
fi
- if [ "$IS_FORK" = "true" ]; then
- echo "Skipping Codex review for fork PR."
+ # Fork PRs run untrusted code with secrets present, so the automatic
+ # pull_request trigger never reviews them. A non-empty INPUT_PR_NUMBER
+ # means we arrived via workflow_call (a maintainer /codex comment gated
+ # by check-write-access), so allow forks only on that path.
+ if [ "$IS_FORK" = "true" ] && [ -z "$INPUT_PR_NUMBER" ]; then
+ echo "Skipping Codex review for fork PR (automatic trigger)."
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi
+ # PR title/body are attacker-controlled free text. Use an unguessable
+ # per-run delimiter so a fork can't embed a fixed heredoc terminator to
+ # inject extra outputs — e.g. is_fork=false (last-write-wins), which
+ # would re-enable the EE checkout and trusted-path settings for forks.
+ RAND=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')
+ TITLE_EOF="TITLE_EOF_${RAND}"
+ BODY_EOF="BODY_EOF_${RAND}"
{
echo "skip=false"
+ echo "is_fork=$IS_FORK"
echo "pr_number=$PR_NUMBER"
echo "base_ref=$BASE_REF"
echo "base_sha=$BASE_SHA"
echo "head_sha=$HEAD_SHA"
echo "pr_author=$PR_AUTHOR"
- echo 'title<> "$GITHUB_OUTPUT"
- name: Checkout repository
@@ -130,9 +142,17 @@ jobs:
with:
ref: refs/pull/${{ steps.pr.outputs.pr_number }}/merge
fetch-depth: 1
+ # Don't persist github.token in .git/config: the review agent can read
+ # the checkout, and on the fork path that token (issue/PR write) would
+ # otherwise be exfiltratable. All later git ops target the public origin
+ # and need no auth; EE checkout and gh use their own explicit tokens.
+ persist-credentials: false
+ # Never expose the EE private-repo token to untrusted fork code. Skipping
+ # this step leaves steps.ee.outputs.available empty, so the EE checkout and
+ # substitution steps below are skipped too.
- name: Check EE access
- if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
+ if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' && steps.pr.outputs.is_fork != 'true'
id: ee
env:
EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
@@ -166,7 +186,7 @@ jobs:
- name: Install Codex CLI
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
- run: npm install --global @openai/codex@0.128.0
+ run: npm install --global @openai/codex@0.144.1
- name: Configure Codex auth
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
@@ -206,9 +226,12 @@ jobs:
REPO: ${{ github.repository }}
PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
run: |
+ # Write outside the checkout: on the fork path the merge tree is
+ # attacker-controlled, and a committed symlink at this path would
+ # redirect the write.
gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" \
--jq '[.[] | {user: .user.login, created_at: .created_at, body: (.body | .[:4000])}] | sort_by(.created_at) | .[-20:]' \
- > prior-comments.json || echo "[]" > prior-comments.json
+ > "$RUNNER_TEMP/prior-comments.json" || echo "[]" > "$RUNNER_TEMP/prior-comments.json"
- name: Write Codex review context
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
@@ -222,9 +245,9 @@ jobs:
PR_AUTHOR: ${{ steps.pr.outputs.pr_author }}
EXTRA_PROMPT: ${{ inputs.extra_prompt }}
run: |
- mkdir -p .github/codex
node <<'NODE'
const fs = require('fs');
+ const tmp = process.env.RUNNER_TEMP;
const lines = [
`Repository: ${process.env.PR_REPOSITORY}`,
`PR number: ${process.env.PR_NUMBER}`,
@@ -254,9 +277,9 @@ jobs:
if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) {
lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim());
}
- if (fs.existsSync('prior-comments.json')) {
+ if (fs.existsSync(`${tmp}/prior-comments.json`)) {
try {
- const comments = JSON.parse(fs.readFileSync('prior-comments.json', 'utf8'));
+ const comments = JSON.parse(fs.readFileSync(`${tmp}/prior-comments.json`, 'utf8'));
if (Array.isArray(comments) && comments.length > 0) {
lines.push(
'',
@@ -271,19 +294,39 @@ jobs:
}
} catch (_) {}
}
- fs.writeFileSync('.github/codex/pr-review-context.md', `${lines.join('\n')}\n`);
+ fs.writeFileSync(`${tmp}/pr-review-context.md`, `${lines.join('\n')}\n`);
NODE
- name: Run Codex review
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
+ env:
+ PR_IS_FORK: ${{ steps.pr.outputs.is_fork }}
+ PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
run: |
- cat REVIEW.md .github/codex/pr-review.prompt.md > /tmp/codex-prompt.md
+ if [ "$PR_IS_FORK" = "true" ]; then
+ # Fork code is untrusted. Read the review policy/prompt from the base
+ # ref (git show) rather than the attacker-controlled merge checkout,
+ # so a malicious fork can't rewrite the reviewer's own instructions,
+ # and run in a network-disabled sandbox to block secret exfiltration.
+ git show "origin/$PR_BASE_REF:REVIEW.md" > /tmp/codex-prompt.md
+ git show "origin/$PR_BASE_REF:.github/codex/pr-review.prompt.md" >> /tmp/codex-prompt.md
+ SANDBOX_MODE=workspace-write
+ else
+ cat REVIEW.md .github/codex/pr-review.prompt.md > /tmp/codex-prompt.md
+ SANDBOX_MODE=danger-full-access
+ fi
+ # The context file lives in RUNNER_TEMP (outside the attacker-controlled
+ # checkout); tell the agent its absolute path.
+ printf '\nReview context file (absolute path): %s\n' "$RUNNER_TEMP/pr-review-context.md" >> /tmp/codex-prompt.md
+ # Write the final message outside the checkout too: a fork could commit
+ # codex-final-message.md as a symlink and redirect this write to overwrite
+ # e.g. a GitHub Action's index.js, which then runs with our credentials.
codex exec \
-C "$GITHUB_WORKSPACE" \
- -m gpt-5.5 \
+ -m gpt-5.6-sol \
-c 'model_reasoning_effort="xhigh"' \
- -s danger-full-access \
- -o codex-final-message.md \
+ -s "$SANDBOX_MODE" \
+ -o "$RUNNER_TEMP/codex-final-message.md" \
- < /tmp/codex-prompt.md
- name: Post Codex review comment
@@ -291,20 +334,52 @@ jobs:
uses: actions/github-script@v7
env:
PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
+ GH_JOB_TOKEN: ${{ github.token }}
with:
github-token: ${{ github.token }}
script: |
const fs = require('fs');
- const path = `${process.env.GITHUB_WORKSPACE}/codex-final-message.md`;
+ const path = `${process.env.RUNNER_TEMP}/codex-final-message.md`;
if (!fs.existsSync(path)) {
core.info('Codex did not produce a final message; skipping PR comment.');
return;
}
- const body = fs.readFileSync(path, 'utf8').trim();
+ let body = fs.readFileSync(path, 'utf8').trim();
if (!body) {
core.info('Codex final message was empty; skipping PR comment.');
return;
}
+ // Defense-in-depth for fork reviews: the model call needs the provider
+ // credential in the env, and the posted comment bypasses Actions log
+ // masking. Strip any credential (API key, raw auth JSON, nested
+ // tokens) that leaked into the review text before posting.
+ const secrets = [];
+ const addSecret = (v, min) => {
+ if (typeof v === 'string' && v.length >= min) secrets.push(v);
+ };
+ addSecret(process.env.OPENAI_API_KEY, 8);
+ addSecret(process.env.CODEX_AUTH_JSON, 8);
+ addSecret(process.env.GH_JOB_TOKEN, 8);
+ if (process.env.CODEX_AUTH_JSON) {
+ try {
+ const collect = (o) => {
+ if (typeof o === 'string') addSecret(o, 20);
+ else if (Array.isArray(o)) o.forEach(collect);
+ else if (o && typeof o === 'object') Object.values(o).forEach(collect);
+ };
+ collect(JSON.parse(process.env.CODEX_AUTH_JSON));
+ } catch (_) {}
+ }
+ for (const s of [...new Set(secrets)].sort((a, b) => b.length - a.length)) {
+ body = body.split(s).join('[REDACTED]');
+ }
+ body = body.trim();
+ if (!body) {
+ core.info('Codex final message was empty after redaction; skipping PR comment.');
+ return;
+ }
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml
index 3443b7e649..bf9009c800 100644
--- a/.github/workflows/git-commands.yaml
+++ b/.github/workflows/git-commands.yaml
@@ -80,7 +80,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
- toolchain: 1.93.0
+ toolchain: 1.97.0
- name: Install xmlsec and gssapi build-time deps
run: |
diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml
index 12b75932ce..3ff7d8b85c 100644
--- a/.github/workflows/git-sync-test.yml
+++ b/.github/workflows/git-sync-test.yml
@@ -121,7 +121,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
- toolchain: 1.93.0
+ toolchain: 1.97.0
- uses: oven-sh/setup-bun@v2
with:
diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml
index 72553b0d83..03c9599480 100644
--- a/.github/workflows/pi-pr-review.yml
+++ b/.github/workflows/pi-pr-review.yml
@@ -97,24 +97,36 @@ jobs:
IS_FORK="$EVENT_FORK"
PR_AUTHOR="$EVENT_AUTHOR"
fi
- if [ "$IS_FORK" = "true" ]; then
- echo "Skipping Pi review for fork PR."
+ # Fork PRs run untrusted code with secrets present, so the automatic
+ # pull_request trigger never reviews them. A non-empty INPUT_PR_NUMBER
+ # means we arrived via workflow_call (a maintainer /pi comment gated by
+ # check-write-access), so allow forks only on that path.
+ if [ "$IS_FORK" = "true" ] && [ -z "$INPUT_PR_NUMBER" ]; then
+ echo "Skipping Pi review for fork PR (automatic trigger)."
echo "skip=true" >> "$GITHUB_OUTPUT"
exit 0
fi
+ # PR title/body are attacker-controlled free text. Use an unguessable
+ # per-run delimiter so a fork can't embed a fixed heredoc terminator to
+ # inject extra outputs — e.g. is_fork=false (last-write-wins), which
+ # would re-enable the EE checkout and trusted-path settings for forks.
+ RAND=$(head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n')
+ TITLE_EOF="TITLE_EOF_${RAND}"
+ BODY_EOF="BODY_EOF_${RAND}"
{
echo "skip=false"
+ echo "is_fork=$IS_FORK"
echo "pr_number=$PR_NUMBER"
echo "base_ref=$BASE_REF"
echo "base_sha=$BASE_SHA"
echo "head_sha=$HEAD_SHA"
echo "pr_author=$PR_AUTHOR"
- echo 'title<> "$GITHUB_OUTPUT"
- name: Checkout repository
@@ -123,9 +135,17 @@ jobs:
with:
ref: refs/pull/${{ steps.pr.outputs.pr_number }}/merge
fetch-depth: 1
+ # Don't persist github.token in .git/config: the review agent can read
+ # the checkout, and on the fork path that token (issue/PR write) would
+ # otherwise be exfiltratable. All later git ops target the public origin
+ # and need no auth; EE checkout and gh use their own explicit tokens.
+ persist-credentials: false
+ # Never expose the EE private-repo token to untrusted fork code. Skipping
+ # this step leaves steps.ee.outputs.available empty, so the EE checkout and
+ # substitution steps below are skipped too.
- name: Check EE access
- if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
+ if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true' && steps.pr.outputs.is_fork != 'true'
id: ee
env:
EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
@@ -178,9 +198,12 @@ jobs:
REPO: ${{ github.repository }}
PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
run: |
+ # Write outside the checkout: on the fork path the merge tree is
+ # attacker-controlled, and a committed symlink at this path would
+ # redirect the write.
gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" \
--jq '[.[] | {user: .user.login, created_at: .created_at, body: (.body | .[:4000])}] | sort_by(.created_at) | .[-20:]' \
- > prior-comments.json || echo "[]" > prior-comments.json
+ > "$RUNNER_TEMP/prior-comments.json" || echo "[]" > "$RUNNER_TEMP/prior-comments.json"
- name: Write Pi review context
if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
@@ -194,9 +217,9 @@ jobs:
PR_AUTHOR: ${{ steps.pr.outputs.pr_author }}
EXTRA_PROMPT: ${{ inputs.extra_prompt }}
run: |
- mkdir -p .github/pi
node <<'NODE'
const fs = require('fs');
+ const tmp = process.env.RUNNER_TEMP;
const lines = [
`Repository: ${process.env.PR_REPOSITORY}`,
`PR number: ${process.env.PR_NUMBER}`,
@@ -226,9 +249,9 @@ jobs:
if (process.env.EXTRA_PROMPT && process.env.EXTRA_PROMPT.trim()) {
lines.push('', 'Additional reviewer instructions:', process.env.EXTRA_PROMPT.trim());
}
- if (fs.existsSync('prior-comments.json')) {
+ if (fs.existsSync(`${tmp}/prior-comments.json`)) {
try {
- const comments = JSON.parse(fs.readFileSync('prior-comments.json', 'utf8'));
+ const comments = JSON.parse(fs.readFileSync(`${tmp}/prior-comments.json`, 'utf8'));
if (Array.isArray(comments) && comments.length > 0) {
lines.push(
'',
@@ -243,7 +266,7 @@ jobs:
}
} catch (_) {}
}
- fs.writeFileSync('.github/pi/pr-review-context.md', `${lines.join('\n')}\n`);
+ fs.writeFileSync(`${tmp}/pr-review-context.md`, `${lines.join('\n')}\n`);
NODE
- name: Run Pi review
@@ -251,16 +274,69 @@ jobs:
env:
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
PI_SKIP_VERSION_CHECK: '1'
+ PR_IS_FORK: ${{ steps.pr.outputs.is_fork }}
+ PR_BASE_REF: ${{ steps.pr.outputs.base_ref }}
+ PR_BASE_SHA: ${{ steps.pr.outputs.base_sha }}
+ PR_HEAD_SHA: ${{ steps.pr.outputs.head_sha }}
run: |
set -o pipefail
- cat REVIEW.md .github/pi/pr-review.prompt.md > /tmp/pi-prompt.md
+ PI_HARDEN_FLAGS=()
+ # Keep generated files (final message, events, context) outside the
+ # checkout: on the fork path a committed symlink at any of these paths
+ # would redirect our write and could overwrite an action's code that
+ # then runs with our credentials. RUNNER_TEMP is outside the checkout.
+ OUT_DIR="$RUNNER_TEMP"
+ CTX="$RUNNER_TEMP/pr-review-context.md"
+ if [ "$PR_IS_FORK" = "true" ]; then
+ # Fork code is untrusted. Read the review policy/prompt from the base
+ # ref (git show) rather than the attacker-controlled merge checkout,
+ # so a malicious fork can't rewrite the reviewer's own instructions,
+ # and drop the bash tool so the agent has no shell to exfiltrate with.
+ git show "origin/$PR_BASE_REF:REVIEW.md" > /tmp/pi-prompt.md
+ git show "origin/$PR_BASE_REF:.github/pi/pr-review.prompt.md" >> /tmp/pi-prompt.md
+ PI_TOOLS=read,grep,find,ls
+
+ # The agent has no shell, so pre-compute the diff (base...head SHAs are
+ # trusted) into the context file it reads. It may still read fork files
+ # by absolute path for extra context — reads are safe.
+ {
+ echo ""
+ echo "## Pre-computed review diff (base...head)"
+ echo "You have no shell. The full diff is below. The repository checkout"
+ echo "is at $GITHUB_WORKSPACE — you may read files there by absolute path."
+ echo '```diff'
+ git -C "$GITHUB_WORKSPACE" diff --unified=0 "$PR_BASE_SHA...$PR_HEAD_SHA"
+ echo '```'
+ } >> "$CTX"
+
+ # Pi resolves ALL project config from /.pi (settings/packages,
+ # extensions, skills, themes, prompts, SYSTEM.md); inside the fork
+ # checkout a fork could inject any to run code or rewrite our system
+ # prompt. Discovery is cwd-based, so run from a fresh empty dir.
+ PI_WORKDIR=$(mktemp -d)
+ cd "$PI_WORKDIR"
+
+ # Belt-and-suspenders on top of the isolated cwd: refuse discovery of
+ # extensions/skills/templates/themes/context-files, and PI_OFFLINE=1 to
+ # block any startup network op or package install. PI_OFFLINE gates only
+ # startup network ops, not the provider inference call.
+ PI_HARDEN_FLAGS=(--no-extensions --no-skills --no-prompt-templates --no-themes --no-context-files)
+ export PI_OFFLINE=1
+ else
+ cat REVIEW.md .github/pi/pr-review.prompt.md > /tmp/pi-prompt.md
+ PI_TOOLS=read,grep,find,ls,bash
+ fi
+ # The context file lives in RUNNER_TEMP (outside the checkout); tell the
+ # agent its absolute path.
+ printf '\nReview context file (absolute path): %s\n' "$CTX" >> /tmp/pi-prompt.md
pi -p \
--provider deepseek \
--model deepseek-v4-pro \
- --tools read,grep,find,ls,bash \
+ --tools "$PI_TOOLS" \
+ "${PI_HARDEN_FLAGS[@]}" \
--mode json \
< /tmp/pi-prompt.md \
- | tee pi-events.jsonl \
+ | tee "$OUT_DIR/pi-events.jsonl" \
| jq -rc --unbuffered '
if .type == "agent_start" then "🤖 pi agent started"
elif .type == "turn_start" then "── turn ──"
@@ -288,27 +364,43 @@ jobs:
| map(select(.role == "assistant"))
| last
| (.content[]? | select(.type == "text") | .text)
- ' pi-events.jsonl > pi-final-message.md
+ ' "$OUT_DIR/pi-events.jsonl" > "$OUT_DIR/pi-final-message.md"
- name: Post Pi review comment
if: steps.pi_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
uses: actions/github-script@v7
env:
PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
+ DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+ GH_JOB_TOKEN: ${{ github.token }}
with:
github-token: ${{ github.token }}
script: |
const fs = require('fs');
- const path = `${process.env.GITHUB_WORKSPACE}/pi-final-message.md`;
+ const path = `${process.env.RUNNER_TEMP}/pi-final-message.md`;
if (!fs.existsSync(path)) {
core.info('Pi did not produce a final message; skipping PR comment.');
return;
}
- const body = fs.readFileSync(path, 'utf8').trim();
+ let body = fs.readFileSync(path, 'utf8').trim();
if (!body) {
core.info('Pi final message was empty; skipping PR comment.');
return;
}
+ // Defense-in-depth for fork reviews: the model call needs the provider
+ // credential in the environment (readable via /proc/self/environ), and
+ // the posted comment is an exfiltration channel that bypasses GitHub
+ // Actions log masking. Strip the credential if it leaked into the text.
+ for (const s of [process.env.DEEPSEEK_API_KEY, process.env.GH_JOB_TOKEN]) {
+ if (typeof s === 'string' && s.length >= 8) {
+ body = body.split(s).join('[REDACTED]');
+ }
+ }
+ body = body.trim();
+ if (!body) {
+ core.info('Pi final message was empty after redaction; skipping PR comment.');
+ return;
+ }
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml
index f1745d03fd..019de201e6 100644
--- a/.github/workflows/publish_windows_worker.yml
+++ b/.github/workflows/publish_windows_worker.yml
@@ -35,7 +35,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
- toolchain: 1.93.0
+ toolchain: 1.97.0
- name: Substitute EE code
shell: bash
diff --git a/AGENTS.md b/AGENTS.md
index 83b68f7d36..fa7889c4c2 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -16,7 +16,7 @@ Open-source platform for internal tools, workflows, API integrations, background
- **Backend patterns**: use the `rust-backend` skill when writing Rust code
- **Frontend patterns**: use the `svelte-frontend` skill when writing Svelte code. Do NOT edit svelte files unless you have read that skill.
- **Frontend UUIDs**: do not call `crypto.randomUUID()` in frontend code. Import `randomUUID` from `$lib/utils/uuid` instead.
-- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi.
+- **Code review**: review the current PR or branch against the shared review policy in `REVIEW.md` (severity triage, public-surface checklist, AGENTS.md compliance, test-coverage assessment). The skill at `.agents/skills/local-review/SKILL.md` orchestrates it. All three CLIs auto-discover the same SKILL — Claude reads `.claude/skills/` (symlinked to the canonical `.agents/skills/` file), Codex and Pi read `.agents/skills/` directly. Invoke with `/local-review` in Claude Code, `$local-review` (or `/skills` selector) in Codex, or `pi --skill local-review` / `/skill:local-review` in Pi. For a Codex-driven pass that mirrors the `codex-pr-review` GitHub action against your unpushed work (committed + uncommitted) before you push, use `/local-review-codex` (`.agents/skills/local-review-codex/`) — same `REVIEW.md` policy, `gpt-5.6-sol`, `xhigh` reasoning; requires the `codex` CLI >= 0.144.1.
- **Domain guides**: `.claude/skills/native-trigger/` and `frontend/tutorial-system-guide.mdc`
- **Brand/UI guidelines**: `frontend/brand-guidelines.md`
- **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5c7cea1e37..51542f3ea4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,90 @@
# Changelog
+## [1.757.0](https://github.com/windmill-labs/windmill/compare/v1.756.1...v1.757.0) (2026-07-14)
+
+
+### Features
+
+* **saml:** add ALLOW_PRIVATE_SAML_METADATA_URLS SSRF bypass ([#10077](https://github.com/windmill-labs/windmill/issues/10077)) ([851e309](https://github.com/windmill-labs/windmill/commit/851e30914e3172dad2a3ccaa6951a3ffe1e67495))
+
+
+### Bug Fixes
+
+* **ai-agent:** don't mark repeated tool calls as failed in flow graph ([#10075](https://github.com/windmill-labs/windmill/issues/10075)) ([207ce86](https://github.com/windmill-labs/windmill/commit/207ce8649cf7026c62eea2b1b2f462c7df8c4e5a))
+
+## [1.756.1](https://github.com/windmill-labs/windmill/compare/v1.756.0...v1.756.1) (2026-07-14)
+
+
+### Bug Fixes
+
+* **apps:** cover script/flow component outputs in deployed-app S3 provenance gate ([#10070](https://github.com/windmill-labs/windmill/issues/10070)) ([710a13a](https://github.com/windmill-labs/windmill/commit/710a13a59d511614839f70e445db736e436550d7))
+
+## [1.756.0](https://github.com/windmill-labs/windmill/compare/v1.755.0...v1.756.0) (2026-07-12)
+
+
+### Features
+
+* **triggers:** serve binary HTTP-route responses via base64 transfer encoding ([#10058](https://github.com/windmill-labs/windmill/issues/10058)) ([29f4cd4](https://github.com/windmill-labs/windmill/commit/29f4cd4b6f58a29b83b84a7a9b8a439d20ade00e)), closes [#5986](https://github.com/windmill-labs/windmill/issues/5986)
+
+
+### Bug Fixes
+
+* replicate all secrets on fork when external backend is configured ([#10060](https://github.com/windmill-labs/windmill/issues/10060)) ([92b7f37](https://github.com/windmill-labs/windmill/commit/92b7f375a90de2f78565ca06a13c79ff04eda44d))
+* **sessions:** sync AI-session preview with workspace edits + stop phantom autosave (WIN-2160) ([#10061](https://github.com/windmill-labs/windmill/issues/10061)) ([5cde2d5](https://github.com/windmill-labs/windmill/commit/5cde2d5b6746be9f2d0be3a98ecdaf08777a6395))
+
+## [1.755.0](https://github.com/windmill-labs/windmill/compare/v1.754.0...v1.755.0) (2026-07-11)
+
+
+### Features
+
+* add per-workspace job-retention override ([#10050](https://github.com/windmill-labs/windmill/issues/10050)) ([ff774c4](https://github.com/windmill-labs/windmill/commit/ff774c46bff4bff1c532e512b163225aa7c41c11))
+* **apps:** authorize deployed-app S3 reads on-behalf of the author for logged-in viewers ([#10048](https://github.com/windmill-labs/windmill/issues/10048)) ([1e192f2](https://github.com/windmill-labs/windmill/commit/1e192f2d864b8a4671e900726972737406bc388a))
+* **mcp:** add multi-workspace MCP tokens via the gateway endpoint ([#10043](https://github.com/windmill-labs/windmill/issues/10043)) ([8343203](https://github.com/windmill-labs/windmill/commit/8343203ec2cea28a2ffd5b4ac636497e861fa3ce))
+
+
+### Bug Fixes
+
+* clearer errors on auto-draft save failure (WIN-2157) ([#10053](https://github.com/windmill-labs/windmill/issues/10053)) ([04eb7dd](https://github.com/windmill-labs/windmill/commit/04eb7ddd3906c28bec1711e276473a87c7b9500f))
+* **docker:** pin ansible tool interpreter to a persistent path ([#10054](https://github.com/windmill-labs/windmill/issues/10054)) ([6f49a1f](https://github.com/windmill-labs/windmill/commit/6f49a1f6a904442fcae9bb703f095b0a3ef61268))
+* enforce read authorization when signing S3 objects ([#10049](https://github.com/windmill-labs/windmill/issues/10049)) ([5844c32](https://github.com/windmill-labs/windmill/commit/5844c32ac5d08081b3de7f3d11b8b98eb1e1ad9a))
+* **frontend:** don't re-seed empty editor on stale ?new_draft after draft exists ([#10044](https://github.com/windmill-labs/windmill/issues/10044)) ([e668193](https://github.com/windmill-labs/windmill/commit/e668193a93b4a7df50459b31f2dc5f9a9b23d0fe))
+* **frontend:** keep draft autosave alive after AI-session round-trip ([#10052](https://github.com/windmill-labs/windmill/issues/10052)) ([7d02d9a](https://github.com/windmill-labs/windmill/commit/7d02d9a1e47760f287a71f6e09cd6fe45efb5635))
+* **frontend:** mint draft path for new SDK builder items so autosave attaches ([#10056](https://github.com/windmill-labs/windmill/issues/10056)) ([a89b896](https://github.com/windmill-labs/windmill/commit/a89b896ce5638f42f334055f9ffe6971b047aa84))
+* **frontend:** show nested restart button for subflows nested in containers ([#10042](https://github.com/windmill-labs/windmill/issues/10042)) ([3b07817](https://github.com/windmill-labs/windmill/commit/3b0781761b70667c5961bcb15d41c907716fa9e7))
+* **frontend:** show optimistic user message and fork-creation label before beforeSend ([#10037](https://github.com/windmill-labs/windmill/issues/10037)) ([1c88242](https://github.com/windmill-labs/windmill/commit/1c88242849a02b927f59e0a67c4b4707371b784f))
+* keep agent-worker server job-completed processor alive & self-healing ([#10033](https://github.com/windmill-labs/windmill/issues/10033)) ([ab38e14](https://github.com/windmill-labs/windmill/commit/ab38e1418e67be8bcc37391bb21d2f86d1ca3fc6))
+
+## [1.754.0](https://github.com/windmill-labs/windmill/compare/v1.753.0...v1.754.0) (2026-07-10)
+
+
+### Features
+
+* add multi-select mode to copilot askUserQuestion ([#10016](https://github.com/windmill-labs/windmill/issues/10016)) ([7569798](https://github.com/windmill-labs/windmill/commit/756979852c3245d06c5c73eb60e9a09fd59635c5))
+
+
+### Bug Fixes
+
+* accept bunnative language in AI chat flow step validation ([#10030](https://github.com/windmill-labs/windmill/issues/10030)) ([5a460db](https://github.com/windmill-labs/windmill/commit/5a460dbec6e2b81e01aa2c36cb504dde4ff6b24a))
+* **backend:** propagate script timeout when restarting perpetual scripts ([#10029](https://github.com/windmill-labs/windmill/issues/10029)) ([6c521e9](https://github.com/windmill-labs/windmill/commit/6c521e9d87724e43ebe3b7fce8b30a3671ef3d89))
+* **frontend:** name the draft in AI chat test-run confirmation ([#10024](https://github.com/windmill-labs/windmill/issues/10024)) ([9036ac7](https://github.com/windmill-labs/windmill/commit/9036ac789f358103b8d639a6b94708c452f52f90))
+* **frontend:** open new script/flow/app in AI session (not-found + friendly tab) ([#10028](https://github.com/windmill-labs/windmill/issues/10028)) ([0353569](https://github.com/windmill-labs/windmill/commit/03535691d607d8a9d1c1fa1c9c284fa3b6051d40))
+* **frontend:** persist forked "Copy of X" script drafts ([#10021](https://github.com/windmill-labs/windmill/issues/10021)) ([c537d45](https://github.com/windmill-labs/windmill/commit/c537d45e4982f30a44026d6644d5340e56f16ef9))
+* **frontend:** persist per-session preview panel resize width ([#10031](https://github.com/windmill-labs/windmill/issues/10031)) ([5387076](https://github.com/windmill-labs/windmill/commit/5387076c1c6fb35a99843aadd2055d3ac381d6cf))
+* **frontend:** scope raw-app, flow and script editors to the session workspace ([#10015](https://github.com/windmill-labs/windmill/issues/10015)) ([c000bbc](https://github.com/windmill-labs/windmill/commit/c000bbca283f5d61cff8a39458764b2b2dd2b58f))
+* resolve fork family/picker for superadmin visiting a non-member workspace ([#10023](https://github.com/windmill-labs/windmill/issues/10023)) ([368fd2d](https://github.com/windmill-labs/windmill/commit/368fd2d9e4b3ffb66e64934d9a622eea291cde5a))
+* scope AI-session flow/script editors to the session workspace ([#10025](https://github.com/windmill-labs/windmill/issues/10025)) ([c5060a1](https://github.com/windmill-labs/windmill/commit/c5060a1e9af5a704e90f92d325abf29626ecd28a))
+* **security:** drop --allow-run from Deno sandbox (GHSA-gj6h-vw66-mr8f) ([#10039](https://github.com/windmill-labs/windmill/issues/10039)) ([c029d6d](https://github.com/windmill-labs/windmill/commit/c029d6dcde44a3d16dee23a80afad920a0535b73))
+* **security:** remove git from Deno sandbox allow-run (GHSA-gj6h-vw66-mr8f) ([#10038](https://github.com/windmill-labs/windmill/issues/10038)) ([689b20a](https://github.com/windmill-labs/windmill/commit/689b20a4704a3dda8d9437b4793c0d16eb1f780f))
+* session preview tab labels, splitter hover, and diff-drawer sizing ([#10008](https://github.com/windmill-labs/windmill/issues/10008)) ([c139eed](https://github.com/windmill-labs/windmill/commit/c139eed631548113b843b466f5505bf6a01f17d3))
+* **sessions:** open test pane when enabling debug so the debug UI is visible ([#9998](https://github.com/windmill-labs/windmill/issues/9998)) ([d7a9b46](https://github.com/windmill-labs/windmill/commit/d7a9b46ab95108c7669b47b7c4be6d8c7964a9e6))
+* sync theme into session page preview iframes on toggle ([#10018](https://github.com/windmill-labs/windmill/issues/10018)) ([3704d00](https://github.com/windmill-labs/windmill/commit/3704d00956dea3b8e562a894d5330e052a753cb3))
+
+
+### Performance Improvements
+
+* index v2_job(parent_job) to speed up run child-job listing ([#10034](https://github.com/windmill-labs/windmill/issues/10034)) ([9feda57](https://github.com/windmill-labs/windmill/commit/9feda57c15bddc7ef481579b73636b88c2a143c5))
+* skip redundant retry-chain job query for successful top-level scripts ([#10035](https://github.com/windmill-labs/windmill/issues/10035)) ([15f9e9b](https://github.com/windmill-labs/windmill/commit/15f9e9b48fc326aa3d776191aabaed22f4c41e74))
+
## [1.753.0](https://github.com/windmill-labs/windmill/compare/v1.752.0...v1.753.0) (2026-07-08)
diff --git a/Dockerfile b/Dockerfile
index a86fb60d2f..f6eb31b0a0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
ARG DEBIAN_IMAGE=debian:trixie-slim
-ARG RUST_IMAGE=rust:1.93-slim-trixie
+ARG RUST_IMAGE=rust:1.97-slim-trixie
FROM debian:trixie-slim AS nsjail
diff --git a/backend/.sqlx/query-056231b872a080a257c034489127e253678631e091b9e4223842648a4475dbbf.json b/backend/.sqlx/query-056231b872a080a257c034489127e253678631e091b9e4223842648a4475dbbf.json
new file mode 100644
index 0000000000..abafce2d82
--- /dev/null
+++ b/backend/.sqlx/query-056231b872a080a257c034489127e253678631e091b9e4223842648a4475dbbf.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT workspace_id as \"workspace_id!\", MIN(completed_at) as oldest\n FROM v2_job_completed\n WHERE workspace_id = ANY($1::text[])\n GROUP BY workspace_id",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "workspace_id!",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 1,
+ "name": "oldest",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ null
+ ]
+ },
+ "hash": "056231b872a080a257c034489127e253678631e091b9e4223842648a4475dbbf"
+}
diff --git a/backend/.sqlx/query-0d373b2600273763c002b1e75f743e1b981f7884df99f033838c4336bea2b051.json b/backend/.sqlx/query-0d373b2600273763c002b1e75f743e1b981f7884df99f033838c4336bea2b051.json
new file mode 100644
index 0000000000..e022f764c8
--- /dev/null
+++ b/backend/.sqlx/query-0d373b2600273763c002b1e75f743e1b981f7884df99f033838c4336bea2b051.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT\n (SELECT MIN(completed_at) FROM v2_job_completed) as true_oldest,\n (SELECT MIN(completed_at) FROM v2_job_completed\n WHERE workspace_id <> ALL($1::text[])) as global_oldest,\n (SELECT COUNT(*) FROM v2_job_completed) as total",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "true_oldest",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 1,
+ "name": "global_oldest",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 2,
+ "name": "total",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null,
+ null,
+ null
+ ]
+ },
+ "hash": "0d373b2600273763c002b1e75f743e1b981f7884df99f033838c4336bea2b051"
+}
diff --git a/backend/.sqlx/query-1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838.json b/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json
similarity index 51%
rename from backend/.sqlx/query-1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838.json
rename to backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json
index be045d9e75..96147d531e 100644
--- a/backend/.sqlx/query-1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838.json
+++ b/backend/.sqlx/query-1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6.json
@@ -1,12 +1,17 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2",
+ "query": "SELECT restart_unless_cancelled, timeout FROM script WHERE hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "restart_unless_cancelled",
"type_info": "Bool"
+ },
+ {
+ "ordinal": 1,
+ "name": "timeout",
+ "type_info": "Int4"
}
],
"parameters": {
@@ -16,8 +21,9 @@
]
},
"nullable": [
+ true,
true
]
},
- "hash": "1d27895aa42ccbb542479b19baefd62790205b529ab0d8af36f18c470e8bb838"
+ "hash": "1debd472c9ffd2fc78877484f93db51f9aabed54f9894eda8ad610053ad76ce6"
}
diff --git a/backend/.sqlx/query-20d62e1937c855cc6887597b3860e991cb539320f4c8321af94416f702108e2e.json b/backend/.sqlx/query-20d62e1937c855cc6887597b3860e991cb539320f4c8321af94416f702108e2e.json
new file mode 100644
index 0000000000..cf55e44fe4
--- /dev/null
+++ b/backend/.sqlx/query-20d62e1937c855cc6887597b3860e991cb539320f4c8321af94416f702108e2e.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT jc.id FROM v2_job_completed jc\n LEFT JOIN v2_job j ON j.id = jc.id\n WHERE jc.workspace_id = $5\n AND jc.completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)\n AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (\n SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL\n )\n ORDER BY jc.completed_at ASC\n LIMIT $2\n FOR UPDATE OF jc SKIP LOCKED\n )\n RETURNING id, completed_at",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "completed_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "UuidArray",
+ "Timestamptz",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "20d62e1937c855cc6887597b3860e991cb539320f4c8321af94416f702108e2e"
+}
diff --git a/backend/.sqlx/query-e4a46d47aee96473a2bd0f1dbdc819520a2ba987fd29859845c2177563ab8cfb.json b/backend/.sqlx/query-67c405ff2bfd68119dbd5e2edc91fde70711b2fb8ec6826411cc7d74687d5bcb.json
similarity index 50%
rename from backend/.sqlx/query-e4a46d47aee96473a2bd0f1dbdc819520a2ba987fd29859845c2177563ab8cfb.json
rename to backend/.sqlx/query-67c405ff2bfd68119dbd5e2edc91fde70711b2fb8ec6826411cc7d74687d5bcb.json
index 18c75f5722..df8b5cce99 100644
--- a/backend/.sqlx/query-e4a46d47aee96473a2bd0f1dbdc819520a2ba987fd29859845c2177563ab8cfb.json
+++ b/backend/.sqlx/query-67c405ff2bfd68119dbd5e2edc91fde70711b2fb8ec6826411cc7d74687d5bcb.json
@@ -1,28 +1,26 @@
{
"db_name": "PostgreSQL",
- "query": "SELECT path, value FROM variable\n WHERE workspace_id = $1 AND is_secret = true AND value != ''",
+ "query": "SELECT id, name FROM workspace WHERE deleted = false ORDER BY name",
"describe": {
"columns": [
{
"ordinal": 0,
- "name": "path",
+ "name": "id",
"type_info": "Varchar"
},
{
"ordinal": 1,
- "name": "value",
+ "name": "name",
"type_info": "Varchar"
}
],
"parameters": {
- "Left": [
- "Text"
- ]
+ "Left": []
},
"nullable": [
false,
false
]
},
- "hash": "e4a46d47aee96473a2bd0f1dbdc819520a2ba987fd29859845c2177563ab8cfb"
+ "hash": "67c405ff2bfd68119dbd5e2edc91fde70711b2fb8ec6826411cc7d74687d5bcb"
}
diff --git a/backend/.sqlx/query-73a3e00e61b33c8ba683ea03f314a229f3dc601b4592d052d3956d5e54d69463.json b/backend/.sqlx/query-73a3e00e61b33c8ba683ea03f314a229f3dc601b4592d052d3956d5e54d69463.json
new file mode 100644
index 0000000000..99621f180f
--- /dev/null
+++ b/backend/.sqlx/query-73a3e00e61b33c8ba683ea03f314a229f3dc601b4592d052d3956d5e54d69463.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT COUNT(*) FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND ($2::text[] IS NULL OR workspace_id NOT IN (\n SELECT u FROM unnest($2::text[]) AS u WHERE u IS NOT NULL\n ))",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "73a3e00e61b33c8ba683ea03f314a229f3dc601b4592d052d3956d5e54d69463"
+}
diff --git a/backend/.sqlx/query-75d1618e12c4dfe5e9632ba8dc45b5aeb572e51ebd0732e6ebf80197acb3e577.json b/backend/.sqlx/query-75d1618e12c4dfe5e9632ba8dc45b5aeb572e51ebd0732e6ebf80197acb3e577.json
new file mode 100644
index 0000000000..817a13cb2f
--- /dev/null
+++ b/backend/.sqlx/query-75d1618e12c4dfe5e9632ba8dc45b5aeb572e51ebd0732e6ebf80197acb3e577.json
@@ -0,0 +1,25 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT EXISTS (\n SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $2\n AND c.started_at > now() - interval '3 hours'\n AND c.result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n AND j.trigger_kind = 'app'\n AND j.trigger = $3\n AND j.created_by = $4\n )",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "exists",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "75d1618e12c4dfe5e9632ba8dc45b5aeb572e51ebd0732e6ebf80197acb3e577"
+}
diff --git a/backend/.sqlx/query-8c1ae47f01c3d5f66faf2f1835c612cea9f6c69d664ceb89094fd544dc63af9d.json b/backend/.sqlx/query-8c1ae47f01c3d5f66faf2f1835c612cea9f6c69d664ceb89094fd544dc63af9d.json
new file mode 100644
index 0000000000..1f90b91084
--- /dev/null
+++ b/backend/.sqlx/query-8c1ae47f01c3d5f66faf2f1835c612cea9f6c69d664ceb89094fd544dc63af9d.json
@@ -0,0 +1,23 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT COUNT(*) FROM v2_job_completed\n WHERE workspace_id = $1\n AND completed_at <= now() - ($2::bigint::text || ' s')::interval",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "count",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "8c1ae47f01c3d5f66faf2f1835c612cea9f6c69d664ceb89094fd544dc63af9d"
+}
diff --git a/backend/.sqlx/query-a2d4a8aedb15e9faf0a2512fa4241a9e9f2d5a56e12a9d6805ca58ff40f61614.json b/backend/.sqlx/query-a2d4a8aedb15e9faf0a2512fa4241a9e9f2d5a56e12a9d6805ca58ff40f61614.json
deleted file mode 100644
index 24e387a783..0000000000
--- a/backend/.sqlx/query-a2d4a8aedb15e9faf0a2512fa4241a9e9f2d5a56e12a9d6805ca58ff40f61614.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND ($3::timestamptz IS NULL OR completed_at >= $3)\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id, completed_at",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "completed_at",
- "type_info": "Timestamptz"
- }
- ],
- "parameters": {
- "Left": [
- "Int8",
- "Int8",
- "Timestamptz"
- ]
- },
- "nullable": [
- false,
- false
- ]
- },
- "hash": "a2d4a8aedb15e9faf0a2512fa4241a9e9f2d5a56e12a9d6805ca58ff40f61614"
-}
diff --git a/backend/.sqlx/query-a460f0ca8f23a8eb9d808b5edd6e0cde0e125f8ed426bd784dd7b92e1d21dfdf.json b/backend/.sqlx/query-a460f0ca8f23a8eb9d808b5edd6e0cde0e125f8ed426bd784dd7b92e1d21dfdf.json
new file mode 100644
index 0000000000..43679e5287
--- /dev/null
+++ b/backend/.sqlx/query-a460f0ca8f23a8eb9d808b5edd6e0cde0e125f8ed426bd784dd7b92e1d21dfdf.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT workspace_id, scopes FROM token WHERE token_hash = $1 AND (expiration > NOW() OR expiration IS NULL)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "workspace_id",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 1,
+ "name": "scopes",
+ "type_info": "TextArray"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ true,
+ true
+ ]
+ },
+ "hash": "a460f0ca8f23a8eb9d808b5edd6e0cde0e125f8ed426bd784dd7b92e1d21dfdf"
+}
diff --git a/backend/.sqlx/query-b326b02adf2245369173fbb1b24386515290c33ea7cedfe2a1092690749ca49c.json b/backend/.sqlx/query-b326b02adf2245369173fbb1b24386515290c33ea7cedfe2a1092690749ca49c.json
new file mode 100644
index 0000000000..5dbd6329b6
--- /dev/null
+++ b/backend/.sqlx/query-b326b02adf2245369173fbb1b24386515290c33ea7cedfe2a1092690749ca49c.json
@@ -0,0 +1,31 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE workspace_id = $4\n AND completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id, completed_at",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "completed_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "Timestamptz",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "b326b02adf2245369173fbb1b24386515290c33ea7cedfe2a1092690749ca49c"
+}
diff --git a/backend/.sqlx/query-b3b06ec52fde4b8264c6307c24b046cd3af17c5ce0d4426153b3065b2faaa781.json b/backend/.sqlx/query-b3b06ec52fde4b8264c6307c24b046cd3af17c5ce0d4426153b3065b2faaa781.json
new file mode 100644
index 0000000000..ce93a35191
--- /dev/null
+++ b/backend/.sqlx/query-b3b06ec52fde4b8264c6307c24b046cd3af17c5ce0d4426153b3065b2faaa781.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT workspace.id, workspace.name\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n WHERE usr.email = $1 AND usr.disabled = false AND workspace.deleted = false\n ORDER BY workspace.name",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Varchar"
+ },
+ {
+ "ordinal": 1,
+ "name": "name",
+ "type_info": "Varchar"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "b3b06ec52fde4b8264c6307c24b046cd3af17c5ce0d4426153b3065b2faaa781"
+}
diff --git a/backend/.sqlx/query-b6de5fbd5fa89e9a8ab8a61982d4f6e02030d8581a7e8c1f0673c7f6dae78fb3.json b/backend/.sqlx/query-b6de5fbd5fa89e9a8ab8a61982d4f6e02030d8581a7e8c1f0673c7f6dae78fb3.json
new file mode 100644
index 0000000000..d05cc5cdd4
--- /dev/null
+++ b/backend/.sqlx/query-b6de5fbd5fa89e9a8ab8a61982d4f6e02030d8581a7e8c1f0673c7f6dae78fb3.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT path FROM variable\n WHERE workspace_id = $1 AND is_secret = true AND value != ''",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "path",
+ "type_info": "Varchar"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "b6de5fbd5fa89e9a8ab8a61982d4f6e02030d8581a7e8c1f0673c7f6dae78fb3"
+}
diff --git a/backend/.sqlx/query-c033a690fde04da79745e850b72fa7cfd861f1dcad88c7ea75a0a8b014ec1f75.json b/backend/.sqlx/query-c033a690fde04da79745e850b72fa7cfd861f1dcad88c7ea75a0a8b014ec1f75.json
deleted file mode 100644
index b2e218e511..0000000000
--- a/backend/.sqlx/query-c033a690fde04da79745e850b72fa7cfd861f1dcad88c7ea75a0a8b014ec1f75.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT jc.id FROM v2_job_completed jc\n LEFT JOIN v2_job j ON j.id = jc.id\n WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND ($4::timestamptz IS NULL OR jc.completed_at >= $4)\n AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (\n SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL\n )\n ORDER BY jc.completed_at ASC\n LIMIT $2\n FOR UPDATE OF jc SKIP LOCKED\n )\n RETURNING id, completed_at",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Uuid"
- },
- {
- "ordinal": 1,
- "name": "completed_at",
- "type_info": "Timestamptz"
- }
- ],
- "parameters": {
- "Left": [
- "Int8",
- "Int8",
- "UuidArray",
- "Timestamptz"
- ]
- },
- "nullable": [
- false,
- false
- ]
- },
- "hash": "c033a690fde04da79745e850b72fa7cfd861f1dcad88c7ea75a0a8b014ec1f75"
-}
diff --git a/backend/.sqlx/query-c89a6cb3b2d5aad2d5e406b5d85149d3ee4cd59ae9d7b154a6d0fcab438b09e6.json b/backend/.sqlx/query-c89a6cb3b2d5aad2d5e406b5d85149d3ee4cd59ae9d7b154a6d0fcab438b09e6.json
new file mode 100644
index 0000000000..ffe341848c
--- /dev/null
+++ b/backend/.sqlx/query-c89a6cb3b2d5aad2d5e406b5d85149d3ee4cd59ae9d7b154a6d0fcab438b09e6.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT jc.id FROM v2_job_completed jc\n LEFT JOIN v2_job j ON j.id = jc.id\n WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)\n AND ($5::text[] IS NULL OR jc.workspace_id NOT IN (\n SELECT u FROM unnest($5::text[]) AS u WHERE u IS NOT NULL\n ))\n AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (\n SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL\n )\n ORDER BY jc.completed_at ASC\n LIMIT $2\n FOR UPDATE OF jc SKIP LOCKED\n )\n RETURNING id, completed_at",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "completed_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "UuidArray",
+ "Timestamptz",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "c89a6cb3b2d5aad2d5e406b5d85149d3ee4cd59ae9d7b154a6d0fcab438b09e6"
+}
diff --git a/backend/.sqlx/query-e9c3e9b9dd6a54c50a7f6639b7922d59d970aa87c0d74ca461f1dea83972ccd9.json b/backend/.sqlx/query-e9c3e9b9dd6a54c50a7f6639b7922d59d970aa87c0d74ca461f1dea83972ccd9.json
new file mode 100644
index 0000000000..10bf1a6061
--- /dev/null
+++ b/backend/.sqlx/query-e9c3e9b9dd6a54c50a7f6639b7922d59d970aa87c0d74ca461f1dea83972ccd9.json
@@ -0,0 +1,31 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "DELETE FROM v2_job_completed\n WHERE id IN (\n SELECT id FROM v2_job_completed\n WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval\n AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)\n AND ($4::text[] IS NULL OR workspace_id NOT IN (\n SELECT u FROM unnest($4::text[]) AS u WHERE u IS NOT NULL\n ))\n ORDER BY completed_at ASC\n LIMIT $2\n FOR UPDATE SKIP LOCKED\n )\n RETURNING id, completed_at",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "completed_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8",
+ "Int8",
+ "Timestamptz",
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "e9c3e9b9dd6a54c50a7f6639b7922d59d970aa87c0d74ca461f1dea83972ccd9"
+}
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md
index 24cf2cb837..96ad6a9d76 100644
--- a/backend/CLAUDE.md
+++ b/backend/CLAUDE.md
@@ -18,6 +18,86 @@
- **Running data pipelines (DuckLake) from source**: see the section below — a plain build
advertises the `duckdb` tag but cannot execute DuckDB scripts and has no working S3 proxy.
+## Cargo features & running the dev backend
+
+The dev backend runs under `cargo watch` and is launched by default with **only
+`--features quickjs`** (see the tmux backend pane). That baseline compiles fast but
+**deliberately omits most functionality** — notably S3/object storage, the S3 proxy, all
+EE code, MCP, and every non-JS language runtime. A running server never gains a feature you
+didn't compile in: feature-gated routes 404 or return a `"requires "` stub. So if
+you touch code behind a feature gate, or need to *exercise* such a feature at runtime, you
+MUST **restart the backend with the appropriate features** for what you're working on.
+
+### Restarting the dev backend with the right features
+
+The backend runs in tmux pane 1 as `cargo watch -x "run --features <…>"`. To restart it with a
+different feature set — scope kills by pid/cwd, **never** `pkill -f target/debug/windmill` (it
+kills every sibling worktree's backend):
+
+1. Stop the current run: `tmux send-keys -t C-c`, then kill *this worktree's*
+ `cargo-watch` pid (find it via `/proc//cwd`).
+2. Relaunch in the same pane so it inherits the shell's `DATABASE_URL` etc.; the pane env's
+ `PORT` may be stale, so set it explicitly:
+ ```bash
+ export PORT=$BACKEND_PORT
+ cargo watch -x "run --features enterprise,private,parquet,quickjs"
+ ```
+3. Wait for `health check completed` in the pane before hitting the API.
+
+cargo-watch only re-runs on a file change, so after an idle/failed run `touch README.md` (from
+`backend/`, where the watch runs) is a cheap retrigger (touching a `.rs` forces a full rebuild).
+
+### What each feature gate does (the ones you'll actually toggle)
+
+`backend/Cargo.toml` `[features]` is the source of truth; this is the practical dev map. Combine
+only what you need — build time scales with the set.
+
+| Feature | Enables | Need it for |
+|---|---|---|
+| `quickjs` | Embedded JS engine for inline JS eval (the default dev baseline). | Keep in every dev set. |
+| `private` | Compiles the `*_ee.rs` files (symlinked from `windmill-ee-private`). Gates **all** EE code, including the real S3 helpers, the S3 proxy, and advanced S3 permission checks. | Any EE code path, S3/object storage. |
+| `enterprise` | EE business logic (autoscaling, SAML hooks, advanced S3 rule **enforcement**, WAP, forks, …). Pulls in `license`. | Running EE features. Advanced S3 permission rules only take effect with this. |
+| `license` | License-key/plan plumbing (`LICENSE_KEY`). Pulled in by `enterprise`. Having the feature compiled does **not** require a license *key* at runtime — CE defaults to a free plan and most EE paths still run keyless. | License-gated behavior. |
+| `parquet` | S3/object-storage support: the `job_helpers/*` and `apps_u/*` S3 endpoints, parquet/CSV preview, workspace large-file storage. Without it those routes return `"requires parquet"`. | Anything touching S3/object storage or datasets. |
+| `duckdb` | DuckDB script executor (also needs the FFI dylib — see above). | DuckDB scripts, DuckLake. |
+| `python` `rust` `php` `java` `ruby` `csharp` `nu` `deno_core` `mysql` `mssql` `bigquery` `oracledb` `rlang` | Each enables that language/DB runtime for job execution. | Running jobs in that language. |
+| `mcp` | MCP gateway routes (baseline `quickjs` does NOT include it → MCP routes 404). | MCP work. |
+| `websocket` `http_trigger` `kafka` `nats` `mqtt_trigger` `sqs_trigger` `gcp_trigger` `azure_trigger` `postgres_trigger` `native_trigger` | Each native trigger kind; none on by default (creating one 404s without its feature). | Working on / exercising that trigger. |
+| `no_auth` | Treats every request as an admin superadmin (`CLOUD_HOSTED`-guarded). | Local auth-free experiments only. |
+
+Convenience bundles (`ce`, `ee`, `oss`, …) exist in `[features]` but are heavy — prefer the
+minimal explicit set for dev.
+
+**Common combinations** (run from `backend/`):
+
+| Goal | `--features` |
+|---|---|
+| Plain dev baseline (JS eval only) | `quickjs` |
+| S3 / object storage / datasets (CE) | `quickjs,private,parquet` |
+| S3 + EE (advanced S3 rules, on-behalf app reads, WAP, forks) | `quickjs,enterprise,private,parquet` |
+| DuckLake / DuckDB (CE) | `quickjs,duckdb,parquet,private` (+ build the FFI) |
+| + Python jobs | append `,python` |
+
+## Workspace object storage in dev — use the local filesystem
+
+For a dev workspace you don't need MinIO/S3: use the built-in **`FilesystemStorage`** large-file
+storage (a root path on local disk). It is intentionally hidden from the settings-UI storage
+dropdown (dev-only), so set it via the API. Requires the backend built with `parquet` (+ `private`
+for the real S3 helpers, + `enterprise` if you want advanced permission rules enforced):
+
+```bash
+curl -X POST "$BASE/api/w//workspaces/edit_large_file_storage_config" \
+ -H "Authorization: Bearer " -H "Content-Type: application/json" \
+ -d '{"large_file_storage":{"type":"FilesystemStorage","root_path":"/abs/writable/dir",
+ "public_resource":false,"advanced_permissions":null,"secondary_storage":{}}}'
+```
+
+Optional `advanced_permissions` (EE) is a list of `{"pattern":"","allow":"read[,write,delete,list]"}`
+rules: admins bypass them, non-admins are confined to matching grants. Uploads/reads then flow
+through the normal `job_helpers/*` (viewer-scoped) and `apps_u/*` (app-author on-behalf) S3
+endpoints. Caveat: direct DuckDB access rejects filesystem stores (`"Filesystem is not supported
+in DuckDB"`) — DuckLake/datatable go through the S3 proxy instead, which works.
+
## Running data pipelines (DuckLake) from source
DuckLake pipelines need **both** the right cargo features **and** the prebuilt DuckDB FFI. A
diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 8a6ff9ad53..e145f1135c 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -815,7 +815,7 @@ dependencies = [
"bytes-utils",
"fastrand",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"percent-encoding",
"pin-project-lite",
"tracing",
@@ -1119,7 +1119,7 @@ dependencies = [
"futures-core",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"percent-encoding",
"pin-project-lite",
@@ -1211,7 +1211,7 @@ dependencies = [
"http 0.2.12",
"http 1.4.2",
"http-body 0.4.6",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"pin-project-lite",
"pin-utils",
@@ -1249,7 +1249,7 @@ dependencies = [
"http 0.2.12",
"http 1.4.2",
"http-body 0.4.6",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"itoa",
"num-integer",
@@ -1306,7 +1306,7 @@ dependencies = [
"bytes",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"itoa",
"matchit 0.7.3",
@@ -1334,7 +1334,7 @@ dependencies = [
"form_urlencoded",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-util",
@@ -1367,7 +1367,7 @@ dependencies = [
"bytes",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"mime",
"pin-project-lite",
@@ -1386,7 +1386,7 @@ dependencies = [
"bytes",
"futures-core",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"mime",
"pin-project-lite",
@@ -1890,18 +1890,18 @@ dependencies = [
[[package]]
name = "bytemuck"
-version = "1.25.0"
+version = "1.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
dependencies = [
"bytemuck_derive",
]
[[package]]
name = "bytemuck_derive"
-version = "1.10.2"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5"
dependencies = [
"proc-macro2",
"quote",
@@ -2057,9 +2057,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.66"
+version = "1.2.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996"
+checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -4622,7 +4622,7 @@ dependencies = [
"futures-core",
"futures-sink",
"nanorand",
- "spin 0.9.8",
+ "spin 0.9.9",
]
[[package]]
@@ -5236,9 +5236,9 @@ dependencies = [
[[package]]
name = "gosyn"
-version = "0.2.11"
+version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1657682b1c3f63ece1fe5b60fc6c5f5923612a20d19f6af38ce79eaf361e3"
+checksum = "938fc2d49f0620e342cf316c6775a1c6722124755cbaab3211cd3dcdce6157c6"
dependencies = [
"anyhow",
"strum",
@@ -5599,9 +5599,9 @@ dependencies = [
[[package]]
name = "http-body"
-version = "1.0.1"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http 1.4.2",
@@ -5609,14 +5609,14 @@ dependencies = [
[[package]]
name = "http-body-util"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
dependencies = [
"bytes",
"futures-core",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"pin-project-lite",
]
@@ -5703,7 +5703,7 @@ dependencies = [
"futures-core",
"h2 0.4.15",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"httparse",
"httpdate",
"itoa",
@@ -5858,13 +5858,13 @@ dependencies = [
"futures-channel",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"hyper 1.10.1",
"ipnet",
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2 0.6.4",
+ "socket2 0.6.5",
"system-configuration",
"tokio",
"tower-service",
@@ -6103,7 +6103,7 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222"
dependencies = [
- "socket2 0.6.4",
+ "socket2 0.6.5",
"widestring",
"windows-registry",
"windows-result 0.4.1",
@@ -6401,7 +6401,7 @@ dependencies = [
"futures",
"home",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-http-proxy",
@@ -6497,7 +6497,7 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
dependencies = [
- "spin 0.9.8",
+ "spin 0.9.9",
]
[[package]]
@@ -6757,9 +6757,9 @@ dependencies = [
[[package]]
name = "lru"
-version = "0.18.0"
+version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9"
+checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6"
dependencies = [
"hashbrown 0.17.1",
]
@@ -7137,9 +7137,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.2.1"
+version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
+checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
@@ -7201,7 +7201,7 @@ dependencies = [
"httparse",
"memchr",
"mime",
- "spin 0.9.8",
+ "spin 0.9.9",
"version_check",
]
@@ -7243,14 +7243,14 @@ dependencies = [
"futures-sink",
"futures-util",
"keyed_priority_queue",
- "lru 0.18.0",
+ "lru 0.18.1",
"mysql_common",
"native-tls",
"pem 3.0.6",
"percent-encoding",
"rand 0.10.2",
"serde",
- "socket2 0.6.4",
+ "socket2 0.6.5",
"thiserror 2.0.18",
"tokio",
"tokio-native-tls",
@@ -8986,7 +8986,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.3",
"rustls 0.23.35",
- "socket2 0.6.4",
+ "socket2 0.6.5",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -9025,7 +9025,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.6.4",
+ "socket2 0.6.5",
"tracing",
"windows-sys 0.61.2",
]
@@ -9370,9 +9370,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.12.4"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
+checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
dependencies = [
"aho-corasick",
"memchr",
@@ -9382,9 +9382,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.14"
+version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db"
dependencies = [
"aho-corasick",
"memchr",
@@ -9441,7 +9441,7 @@ dependencies = [
"futures-util",
"h2 0.4.15",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-rustls 0.27.9",
@@ -9489,7 +9489,7 @@ dependencies = [
"futures-util",
"h2 0.4.15",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-rustls 0.27.9",
@@ -9653,7 +9653,7 @@ dependencies = [
"chrono",
"futures",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"oauth2",
"pastey",
@@ -10160,9 +10160,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "ryu-js"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd29631678d6fb0903b69223673e122c32e9ae559d0960a38d574695ebc0ea15"
+checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714"
[[package]]
name = "safetensors"
@@ -10663,9 +10663,9 @@ dependencies = [
[[package]]
name = "sha1"
-version = "0.10.6"
+version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
+checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
@@ -10869,9 +10869,9 @@ dependencies = [
[[package]]
name = "socket2"
-version = "0.6.4"
+version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -10914,9 +10914,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
-version = "0.9.8"
+version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
dependencies = [
"lock_api",
]
@@ -11196,7 +11196,7 @@ checksum = "39f24a9b78c40b90817bbcd1821c74ddfd74916aadd29403d001532a9195532d"
dependencies = [
"bytes",
"futures-util",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"pin-project-lite",
]
@@ -12107,9 +12107,9 @@ dependencies = [
[[package]]
name = "thread_local"
-version = "1.1.9"
+version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
+checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"
dependencies = [
"cfg-if",
]
@@ -12248,9 +12248,9 @@ dependencies = [
[[package]]
name = "tinyvec"
-version = "1.11.0"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
dependencies = [
"tinyvec_macros",
]
@@ -12383,7 +12383,7 @@ dependencies = [
"postgres-protocol",
"postgres-types",
"rand 0.9.0",
- "socket2 0.6.4",
+ "socket2 0.6.5",
"tokio",
"tokio-util",
"whoami",
@@ -12587,7 +12587,7 @@ dependencies = [
"indexmap 2.14.0",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
- "winnow 1.0.3",
+ "winnow 1.0.4",
]
[[package]]
@@ -12596,7 +12596,7 @@ version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
- "winnow 1.0.3",
+ "winnow 1.0.4",
]
[[package]]
@@ -12613,7 +12613,7 @@ dependencies = [
"flate2",
"h2 0.4.15",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-timeout",
@@ -12645,7 +12645,7 @@ dependencies = [
"bytes",
"h2 0.4.15",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper 1.10.1",
"hyper-timeout",
@@ -12732,7 +12732,7 @@ dependencies = [
"futures-core",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"mime",
"pin-project-lite",
@@ -13350,9 +13350,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.23.4"
+version = "1.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53"
+checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
dependencies = [
"getrandom 0.4.3",
"js-sys",
@@ -13745,7 +13745,7 @@ dependencies = [
[[package]]
name = "windmill"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-nats",
@@ -13827,7 +13827,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"async-stream",
"async-trait",
@@ -13860,7 +13860,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -13873,7 +13873,7 @@ dependencies = [
[[package]]
name = "windmill-api"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"argon2",
@@ -14011,7 +14011,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14034,7 +14034,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14049,7 +14049,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14075,7 +14075,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -14085,7 +14085,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14102,7 +14102,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -14124,7 +14124,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14147,7 +14147,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14163,7 +14163,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14184,7 +14184,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14205,7 +14205,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14219,7 +14219,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14254,7 +14254,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14279,7 +14279,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"flate2",
@@ -14297,7 +14297,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14319,7 +14319,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14339,7 +14339,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14376,7 +14376,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14404,7 +14404,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"lazy_static",
"serde",
@@ -14416,7 +14416,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -14441,7 +14441,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14455,7 +14455,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14490,7 +14490,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"chrono",
"lazy_static",
@@ -14504,7 +14504,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14523,7 +14523,7 @@ dependencies = [
[[package]]
name = "windmill-common"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -14625,7 +14625,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -14644,7 +14644,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"regex",
"serde",
@@ -14659,7 +14659,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -14683,7 +14683,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"futures",
@@ -14700,7 +14700,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -14716,7 +14716,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -14737,7 +14737,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -14768,7 +14768,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -14793,7 +14793,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-stream",
@@ -14827,7 +14827,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"futures",
@@ -14845,7 +14845,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -14854,7 +14854,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14866,7 +14866,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14878,7 +14878,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"gosyn",
@@ -14890,7 +14890,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14902,7 +14902,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14914,7 +14914,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -14925,7 +14925,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14936,7 +14936,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14948,7 +14948,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -14959,7 +14959,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -14981,7 +14981,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14993,7 +14993,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15007,7 +15007,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15024,7 +15024,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15037,7 +15037,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde",
@@ -15049,7 +15049,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15067,7 +15067,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -15083,7 +15083,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15099,7 +15099,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde",
@@ -15110,7 +15110,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15149,7 +15149,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"const_format",
@@ -15188,7 +15188,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -15199,7 +15199,7 @@ dependencies = [
[[package]]
name = "windmill-store"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15233,7 +15233,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15257,7 +15257,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15290,7 +15290,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15323,7 +15323,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15343,7 +15343,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15377,7 +15377,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15413,7 +15413,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15436,7 +15436,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15460,7 +15460,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15484,7 +15484,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15519,7 +15519,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15547,7 +15547,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15572,7 +15572,7 @@ dependencies = [
[[package]]
name = "windmill-types"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"bitflags 2.13.0",
@@ -15591,7 +15591,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -15701,7 +15701,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"bytes",
"futures",
@@ -16301,9 +16301,9 @@ dependencies = [
[[package]]
name = "winnow"
-version = "1.0.3"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
+checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81"
dependencies = [
"memchr",
]
@@ -16427,18 +16427,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.53"
+version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1"
+checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.53"
+version = "0.8.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71"
+checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
dependencies = [
"proc-macro2",
"quote",
@@ -16519,15 +16519,15 @@ dependencies = [
[[package]]
name = "zlib-rs"
-version = "0.6.5"
+version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5"
+checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5"
[[package]]
name = "zmij"
-version = "1.0.21"
+version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
[[package]]
name = "zstd"
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 12ec704c32..e207dffbae 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "windmill"
-version = "1.753.0"
+version = "1.757.0"
authors.workspace = true
edition.workspace = true
@@ -87,7 +87,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
-version = "1.753.0"
+version = "1.757.0"
authors = ["Ruben Fiszel "]
edition = "2021"
diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md
index 9501b5bf0d..0468ebf721 100644
--- a/backend/THREAT_MODEL.md
+++ b/backend/THREAT_MODEL.md
@@ -97,7 +97,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical).
| id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence |
|---|---|---|---|---|---|---|---|---|---|
| T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b |
-| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 |
+| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; SAML IdP metadata URLs are SSRF-validated at load time behind the `ALLOW_PRIVATE_SAML_METADATA_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 |
| T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 |
| T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 |
| T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e |
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
index 03f8422000..7a4ff4906d 100644
--- a/backend/ee-repo-ref.txt
+++ b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-593ad8e171478758e95785f91c5d9548e09957bf
\ No newline at end of file
+25cbc0a7589fd2acd430e5991e4fdd36dde8c215
\ No newline at end of file
diff --git a/backend/migrations/20260710073406_index_v2_job_parent_job.down.sql b/backend/migrations/20260710073406_index_v2_job_parent_job.down.sql
new file mode 100644
index 0000000000..86d52ecae7
--- /dev/null
+++ b/backend/migrations/20260710073406_index_v2_job_parent_job.down.sql
@@ -0,0 +1 @@
+DROP INDEX IF EXISTS ix_v2_job_parent_job;
diff --git a/backend/migrations/20260710073406_index_v2_job_parent_job.up.sql b/backend/migrations/20260710073406_index_v2_job_parent_job.up.sql
new file mode 100644
index 0000000000..b4771e7aa5
--- /dev/null
+++ b/backend/migrations/20260710073406_index_v2_job_parent_job.up.sql
@@ -0,0 +1,11 @@
+-- Partial index for listing a run's child jobs (flow steps, loop iterations,
+-- native-retry attempts, schedule handlers) via the `parent_job = ?` filter on
+-- /jobs/list and /jobs/completed/list. Without it, Postgres walks the whole
+-- workspace (workspace_id, created_at) timeline filtering row-by-row for the
+-- parent. Children of one parent are few, so (parent_job, created_at DESC)
+-- returns them directly and serves both ASC and DESC orderings.
+-- Partial on parent_job IS NOT NULL keeps it small (root jobs are the majority).
+-- Created CONCURRENTLY via the OVERRIDDEN_MIGRATIONS rewrite in windmill-api/src/db.rs.
+CREATE INDEX IF NOT EXISTS ix_v2_job_parent_job
+ ON v2_job (parent_job, created_at DESC)
+ WHERE parent_job IS NOT NULL;
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock
index 4961d0907a..f415ddf6dc 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.lock
+++ b/backend/parsers/windmill-parser-wasm/Cargo.lock
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6272,7 +6272,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"convert_case",
"serde",
@@ -6293,7 +6293,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6305,7 +6305,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6317,7 +6317,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6329,7 +6329,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6341,7 +6341,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6353,7 +6353,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6364,7 +6364,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6375,7 +6375,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6387,7 +6387,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6398,7 +6398,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6420,7 +6420,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6432,7 +6432,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6446,7 +6446,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6463,7 +6463,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde",
@@ -6488,7 +6488,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6522,7 +6522,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6538,7 +6538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6570,7 +6570,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"serde",
@@ -6581,7 +6581,7 @@ dependencies = [
[[package]]
name = "windmill-types"
-version = "1.753.0"
+version = "1.757.0"
dependencies = [
"anyhow",
"bitflags",
diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml
index 5763177b0b..af7aac8ff1 100644
--- a/backend/parsers/windmill-parser-wasm/Cargo.toml
+++ b/backend/parsers/windmill-parser-wasm/Cargo.toml
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
-version = "1.753.0"
+version = "1.757.0"
edition = "2021"
authors = ["Ruben Fiszel "]
diff --git a/backend/src/main.rs b/backend/src/main.rs
index 4115e2a12b..7080ae0c13 100644
--- a/backend/src/main.rs
+++ b/backend/src/main.rs
@@ -56,11 +56,11 @@ use windmill_common::{
PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING,
PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
- RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
- SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING,
- SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING,
- SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
- STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
+ RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING,
+ SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING,
+ SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING,
+ SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING,
+ SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
@@ -124,7 +124,7 @@ use windmill_worker::{
use crate::monitor::{
initial_load, load_disable_password_login, load_fork_workspace_tag_append_fork_suffix,
load_keep_job_dir, load_metrics_debug_enabled, load_preview_tags_override,
- load_require_preexisting_user, load_tag_per_workspace_enabled,
+ load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled,
load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs,
load_workspace_fairness_enabled, load_workspace_fairness_max_percent,
load_workspace_fairness_min_total, monitor_db, reload_app_workspaced_route_setting,
@@ -1881,6 +1881,11 @@ async fn process_notify_event(
}
TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await,
RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await,
+ RETENTION_PERIOD_SECS_OVERRIDES_SETTING => {
+ if let Err(e) = load_retention_period_overrides(db).await {
+ tracing::error!("Error loading per-workspace retention overrides: {e:#}");
+ }
+ }
AUDIT_LOG_RETENTION_DAYS_SETTING => {
reload_audit_log_retention_days_setting(conn).await
}
diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs
index 60b4024c0a..6632e3b3a8 100644
--- a/backend/src/monitor.rs
+++ b/backend/src/monitor.rs
@@ -94,7 +94,8 @@ use windmill_common::{
},
KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE,
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB,
- DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
+ DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
+ JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3,
};
@@ -265,6 +266,12 @@ pub async fn initial_load(
tracing::error!("Error loading preview tags override: {e:#}");
}
+ // Load per-workspace retention overrides before the first cleanup tick so a fresh server
+ // never sweeps globally without honoring configured longer-retention workspaces.
+ if let Err(e) = load_retention_period_overrides(db).await {
+ tracing::error!("Error loading per-workspace retention overrides: {e:#}");
+ }
+
// Workspace fairness (cloud-only). Load the percentage/duration/min knobs
// *before* the enabled flag so that `load_workspace_fairness_enabled` reads
// current values when re-storing the pull queries.
@@ -1339,68 +1346,73 @@ pub async fn delete_expired_items(db: &DB) -> () {
),
}
- let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed);
- if job_retention_secs > 0 {
- let batch_size = *JOB_CLEANUP_BATCH_SIZE;
- let max_batches = *JOB_CLEANUP_MAX_BATCHES;
- let cleanup_start = Instant::now();
- let mut total_deleted = 0u64;
- let mut batch_num = 0i32;
- // Watermark carried across batches so each one resumes after the rows the previous batch
- // already processed instead of re-scanning the (potentially undeletable) oldest prefix.
- let mut completed_at_floor: Option> = None;
+ // Per-workspace retention overrides (EE-only; the cache is always empty on CE). A workspace may
+ // keep jobs LONGER or SHORTER than the instance-wide window. Phase 1 sweeps globally on the
+ // instance window but excludes override workspaces; Phase 2 sweeps each override workspace on its
+ // own window (a sargable `workspace_id = $w` scan). The override count is capped small
+ // (`MAX_RETENTION_OVERRIDE_WORKSPACES`), so Phase 2's per-workspace fan-out stays bounded.
+ //
+ // Deliberate simplicity/scale trade-off: a LONGER or keep-forever override lets that workspace's
+ // old rows accumulate at the front of the completed_at index, and Phase 1's first batch each tick
+ // scans past that retained prefix (an index scan, thanks to the sargable floor — not a Seq Scan)
+ // before reaching a deletable row. This is only material at extreme scale (millions of retained
+ // rows on one busy keep-forever workspace); we accept it rather than carrying a cross-tick
+ // watermark, given overrides are a capped, targeted escape hatch.
+ //
+ // Gate the whole sweep on a confirmed-known override set: if the load never succeeded (e.g. a
+ // startup DB hiccup, or malformed data), the empty cache is "unknown", not "no overrides", and
+ // sweeping globally would delete jobs a longer-retention workspace configured. Retry the load
+ // once here (on CE the flag is already set at startup, so this is a no-op), and skip the whole
+ // job-cleanup phase this tick if still unknown — it runs again shortly.
+ if !JOB_RETENTION_SECS_OVERRIDES_LOADED.load(std::sync::atomic::Ordering::Relaxed) {
+ if let Err(e) = load_retention_period_overrides(db).await {
+ tracing::error!("Error (re)loading per-workspace retention overrides: {e:#}");
+ }
+ }
+ if !JOB_RETENTION_SECS_OVERRIDES_LOADED.load(std::sync::atomic::Ordering::Relaxed) {
+ tracing::error!(
+ "Skipping job retention cleanup this cycle: per-workspace overrides not yet loaded"
+ );
+ } else {
+ let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed);
+ // `load_full` (owned Arc) rather than `load` (Guard): the sweep below holds this across many
+ // `.await`s, and an arc_swap Guard is not meant to be held for long.
+ let retention_overrides = JOB_RETENTION_SECS_OVERRIDES.load_full();
+ let override_workspace_ids: Vec = retention_overrides.keys().cloned().collect();
- // Process batches until no more expired jobs or max batches reached
- loop {
- if max_batches > 0 && batch_num >= max_batches {
- tracing::debug!(
- "Job cleanup: reached max batches limit ({}), will continue next iteration",
- max_batches
- );
- break;
+ // Phase 1: global sweep with the instance window, skipping override workspaces.
+ if job_retention_secs > 0 {
+ run_retention_cleanup(
+ db,
+ job_retention_secs,
+ RetentionScope::GlobalExcluding(&override_workspace_ids),
+ )
+ .await;
+
+ // Clean up concurrency keys separately (not tied to specific job IDs). Kept global on
+ // the instance window — concurrency keys are short-lived and not worth per-workspace
+ // scoping.
+ if let Err(e) = sqlx::query!(
+ "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
+ job_retention_secs
+ )
+ .execute(db)
+ .await
+ {
+ tracing::error!("Error deleting custom concurrency key: {:?}", e);
}
+ }
- // Each batch runs in its own transaction to avoid long-running locks
- let batch_result =
- delete_expired_jobs_batch(db, job_retention_secs, batch_size, completed_at_floor)
+ // Phase 2: each override workspace swept on its own window. A window of 0 means "keep
+ // forever" for that workspace, so it is excluded from Phase 1 above and skipped here. The
+ // override count is capped at MAX_RETENTION_OVERRIDE_WORKSPACES (enforced at write time), so
+ // this loop runs a bounded number of scoped sweeps per pass.
+ for (w_id, retention_secs) in retention_overrides.iter() {
+ if *retention_secs > 0 {
+ run_retention_cleanup(db, *retention_secs, RetentionScope::OnlyWorkspace(w_id))
.await;
-
- match batch_result {
- Ok((deleted_count, max_completed_at)) => {
- if deleted_count == 0 {
- // No more expired jobs to delete
- break;
- }
- completed_at_floor = max_completed_at.or(completed_at_floor);
- total_deleted += deleted_count as u64;
- batch_num += 1;
- }
- Err(e) => {
- tracing::error!("Error in job cleanup batch {}: {:?}", batch_num, e);
- break;
- }
}
}
-
- if total_deleted > 0 {
- tracing::info!(
- "Job cleanup completed: deleted {} jobs in {} batches, took {:?}",
- total_deleted,
- batch_num,
- cleanup_start.elapsed()
- );
- }
-
- // Clean up concurrency keys separately (not tied to specific job IDs)
- if let Err(e) = sqlx::query!(
- "DELETE FROM concurrency_key WHERE ended_at <= now() - ($1::bigint::text || ' s')::interval",
- job_retention_secs
- )
- .execute(db)
- .await
- {
- tracing::error!("Error deleting custom concurrency key: {:?}", e);
- }
}
match windmill_common::trashbin::delete_expired_trash(db).await {
@@ -1546,11 +1558,18 @@ pub async fn check_expiring_tokens(db: &DB) {
///
/// Returns `(jobs deleted in this batch, max completed_at deleted)`. The caller feeds the
/// returned watermark back in as `completed_at_floor` for the next batch.
+///
+/// `only_workspace` and `exclude_workspaces` implement the per-workspace retention override and are
+/// mutually exclusive: Phase 1 passes `exclude_workspaces` (skip override workspaces, sweep the
+/// rest), Phase 2 passes `only_workspace` (sweep just that workspace on its own window). Both `None`
+/// reproduces the plain global sweep exactly. See `run_retention_cleanup` / `delete_expired_items`.
async fn delete_expired_jobs_batch(
db: &DB,
job_retention_secs: i64,
batch_size: i64,
completed_at_floor: Option>,
+ only_workspace: Option<&str>,
+ exclude_workspaces: Option<&[String]>,
) -> error::Result<(usize, Option>)> {
let mut tx = db.begin().await?;
@@ -1571,65 +1590,142 @@ async fn delete_expired_jobs_batch(
// max(completed_at) deleted by the previous batch. Re-applying it as `completed_at >= floor`
// lets each batch resume after the rows the previous batch already processed instead of
// re-scanning them. This matters when the oldest rows are undeletable (children of a
- // still-active root flow): without the floor the `ORDER BY completed_at ASC` scan walks that
- // same protected prefix on every batch, turning a cleanup run quadratic in prefix size.
+ // still-active root flow, or override workspaces excluded from the global sweep): without the
+ // floor the `ORDER BY completed_at ASC` scan walks that same protected/retained prefix on every
+ // batch, turning a cleanup run quadratic in prefix size.
// Floor only ever skips rows the current run already deleted, was protecting, or skip-locked —
// all correctly deferred to the next run, identical to the unbounded scan's semantics.
//
+ // It is applied as `completed_at >= COALESCE($floor, '-infinity')`, NOT `$floor IS NULL OR
+ // completed_at >= $floor`: the `OR ... IS NULL` disjunction is non-sargable, so the planner
+ // cannot use the floor as an index lower bound and falls back to a Seq Scan of the whole table —
+ // walking the entire prefix regardless of the floor. The COALESCE sentinel keeps a single cached
+ // query while making the bound a plain range predicate the completed_at / composite index drives.
+ //
// Use FOR UPDATE SKIP LOCKED to avoid contention between replicas; ORDER BY completed_at
// deletes oldest jobs first.
- let (deleted_jobs, max_completed_at) = if active_root_job_ids.is_empty() {
- // Common case: no old root flow is still running, so nothing is protected and the
- // v2_job join (a PK lookup per candidate) is pure overhead — skip it entirely.
- let rows = sqlx::query!(
- "DELETE FROM v2_job_completed
- WHERE id IN (
- SELECT id FROM v2_job_completed
- WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
- AND ($3::timestamptz IS NULL OR completed_at >= $3)
- ORDER BY completed_at ASC
- LIMIT $2
- FOR UPDATE SKIP LOCKED
- )
- RETURNING id, completed_at",
- job_retention_secs,
- batch_size,
- completed_at_floor,
- )
- .fetch_all(&mut *tx)
- .await?;
- let max = rows.iter().map(|r| r.completed_at).max();
- (rows.into_iter().map(|r| r.id).collect::>(), max)
- } else {
- // Active-root exclusion uses `NOT IN (SELECT ... unnest($3))` rather than `!= ALL($3)`:
- // the subquery form lets the planner build a one-time hashed SubPlan and apply it as a
- // filter on the ordered index scan, giving O(1) membership per candidate instead of a
- // per-row linear array scan (which degrades sharply when many root jobs are active). The
- // `u IS NOT NULL` guard sidesteps NOT IN's null-trap semantics ($3 holds non-null PK ids).
- let rows = sqlx::query!(
- "DELETE FROM v2_job_completed
- WHERE id IN (
- SELECT jc.id FROM v2_job_completed jc
- LEFT JOIN v2_job j ON j.id = jc.id
- WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
- AND ($4::timestamptz IS NULL OR jc.completed_at >= $4)
- AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
- SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
- )
- ORDER BY jc.completed_at ASC
- LIMIT $2
- FOR UPDATE OF jc SKIP LOCKED
- )
- RETURNING id, completed_at",
- job_retention_secs,
- batch_size,
- &active_root_job_ids,
- completed_at_floor,
- )
- .fetch_all(&mut *tx)
- .await?;
- let max = rows.iter().map(|r| r.completed_at).max();
- (rows.into_iter().map(|r| r.id).collect::>(), max)
+ // Two orthogonal choices drive which DELETE we run:
+ // - `only_workspace`: Some => a single-workspace (Phase 2) sweep. We bind `workspace_id = $n`
+ // directly (no `OR $n IS NULL` guard) so the composite `(workspace_id, completed_at)` index
+ // can drive the ordered scan — a sargable equality the OR-form would defeat. `None` => a
+ // global (Phase 1) sweep that instead excludes override workspaces via a hashed `NOT IN
+ // (SELECT ... unnest($exclude))` SubPlan (same one-time-hash trick as the active-root
+ // exclusion below): O(1) membership per candidate, vs `<> ALL($exclude)`'s per-row linear
+ // array scan which degrades sharply once many workspaces have overrides.
+ // - `active_root_job_ids.is_empty()`: skip the `v2_job` join entirely when nothing is
+ // protected (a PK lookup per candidate is pure overhead in the common case).
+ let (deleted_jobs, max_completed_at) = match only_workspace {
+ Some(w_id) if active_root_job_ids.is_empty() => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT id FROM v2_job_completed
+ WHERE workspace_id = $4
+ AND completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)
+ ORDER BY completed_at ASC
+ LIMIT $2
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ completed_at_floor,
+ w_id,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ Some(w_id) => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT jc.id FROM v2_job_completed jc
+ LEFT JOIN v2_job j ON j.id = jc.id
+ WHERE jc.workspace_id = $5
+ AND jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)
+ AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
+ SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
+ )
+ ORDER BY jc.completed_at ASC
+ LIMIT $2
+ FOR UPDATE OF jc SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ &active_root_job_ids,
+ completed_at_floor,
+ w_id,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ None if active_root_job_ids.is_empty() => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT id FROM v2_job_completed
+ WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)
+ AND ($4::text[] IS NULL OR workspace_id NOT IN (
+ SELECT u FROM unnest($4::text[]) AS u WHERE u IS NOT NULL
+ ))
+ ORDER BY completed_at ASC
+ LIMIT $2
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ completed_at_floor,
+ exclude_workspaces,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ None => {
+ // Active-root exclusion uses `NOT IN (SELECT ... unnest($3))` rather than `!= ALL($3)`:
+ // the subquery form lets the planner build a one-time hashed SubPlan and apply it as a
+ // filter on the ordered index scan, giving O(1) membership per candidate instead of a
+ // per-row linear array scan (which degrades sharply when many root jobs are active). The
+ // `u IS NOT NULL` guard sidesteps NOT IN's null-trap semantics ($3 holds non-null PK ids).
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT jc.id FROM v2_job_completed jc
+ LEFT JOIN v2_job j ON j.id = jc.id
+ WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)
+ AND ($5::text[] IS NULL OR jc.workspace_id NOT IN (
+ SELECT u FROM unnest($5::text[]) AS u WHERE u IS NOT NULL
+ ))
+ AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
+ SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
+ )
+ ORDER BY jc.completed_at ASC
+ LIMIT $2
+ FOR UPDATE OF jc SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ &active_root_job_ids,
+ completed_at_floor,
+ exclude_workspaces,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
};
let deleted_count = deleted_jobs.len();
@@ -1704,6 +1800,189 @@ async fn delete_expired_jobs_batch(
Ok((deleted_count, max_completed_at))
}
+/// Which workspaces a retention cleanup run targets.
+#[derive(Debug)]
+enum RetentionScope<'a> {
+ /// Sweep every workspace except the listed ones (they run in their own Phase-2 pass).
+ GlobalExcluding(&'a [String]),
+ /// Sweep only this single workspace, on its own retention window.
+ OnlyWorkspace(&'a str),
+}
+
+/// Drives the batched job-retention delete for a given `retention_secs` window and `scope`.
+/// Preserves the per-run `completed_at_floor` watermark across batches (see
+/// `delete_expired_jobs_batch`). Returns the number of jobs deleted.
+///
+/// `JOB_CLEANUP_MAX_BATCHES` bounds the batches per call, i.e. per scope. A full cleanup cycle can
+/// therefore run up to `(1 + n_override_workspaces) * max_batches` batches; the override count is
+/// capped at `MAX_RETENTION_OVERRIDE_WORKSPACES`, and any residue is picked up on the next tick.
+async fn run_retention_cleanup(db: &DB, retention_secs: i64, scope: RetentionScope<'_>) -> u64 {
+ let (only_workspace, exclude_workspaces): (Option<&str>, Option<&[String]>) = match &scope {
+ // An empty exclusion list binds as NULL so the guard short-circuits to the plain sweep.
+ RetentionScope::GlobalExcluding(ids) => {
+ (None, if ids.is_empty() { None } else { Some(*ids) })
+ }
+ RetentionScope::OnlyWorkspace(w_id) => (Some(*w_id), None),
+ };
+
+ let batch_size = *JOB_CLEANUP_BATCH_SIZE;
+ let max_batches = *JOB_CLEANUP_MAX_BATCHES;
+ let cleanup_start = Instant::now();
+ let mut total_deleted = 0u64;
+ let mut batch_num = 0i32;
+ // Watermark carried across batches so each one resumes after the rows the previous batch
+ // already processed instead of re-scanning the (potentially undeletable) oldest prefix.
+ let mut completed_at_floor: Option> = None;
+
+ // Process batches until no more expired jobs or max batches reached
+ loop {
+ if max_batches > 0 && batch_num >= max_batches {
+ tracing::debug!(
+ "Job cleanup ({scope:?}): reached max batches limit ({max_batches}), will continue next iteration"
+ );
+ break;
+ }
+
+ // Each batch runs in its own transaction to avoid long-running locks
+ let batch_result = delete_expired_jobs_batch(
+ db,
+ retention_secs,
+ batch_size,
+ completed_at_floor,
+ only_workspace,
+ exclude_workspaces,
+ )
+ .await;
+
+ match batch_result {
+ Ok((deleted_count, max_completed_at)) => {
+ if deleted_count == 0 {
+ // No more expired jobs to delete
+ break;
+ }
+ completed_at_floor = max_completed_at.or(completed_at_floor);
+ total_deleted += deleted_count as u64;
+ batch_num += 1;
+ }
+ Err(e) => {
+ tracing::error!("Error in job cleanup batch {batch_num} ({scope:?}): {e:?}");
+ break;
+ }
+ }
+ }
+
+ if total_deleted > 0 {
+ tracing::info!(
+ "Job cleanup completed ({scope:?}): deleted {total_deleted} jobs in {batch_num} batches, took {:?}",
+ cleanup_start.elapsed()
+ );
+ }
+
+ total_deleted
+}
+
+/// Parses the raw `{workspace_id: seconds}` global-setting object into an override map. Returns
+/// `Err` (with the offending workspace) if ANY value is not a non-negative integer, so the caller
+/// can keep the last-good map instead of dropping just that entry — dropping a longer-retention
+/// entry would let the Phase-1 global window delete its jobs, and a negative value would silently
+/// become keep-forever (Phase 2 only sweeps `> 0`).
+#[cfg(feature = "enterprise")]
+fn parse_retention_overrides(
+ map: serde_json::Map,
+) -> std::result::Result, String> {
+ use windmill_common::global_settings::MAX_RETENTION_OVERRIDE_WORKSPACES;
+ if map.len() > MAX_RETENTION_OVERRIDE_WORKSPACES {
+ return Err(format!(
+ "at most {MAX_RETENTION_OVERRIDE_WORKSPACES} per-workspace retention overrides are allowed, got {}",
+ map.len()
+ ));
+ }
+ let mut overrides = std::collections::HashMap::with_capacity(map.len());
+ for (w_id, v) in map {
+ match v.as_i64() {
+ Some(secs) if secs >= 0 => {
+ overrides.insert(w_id, secs);
+ }
+ _ => {
+ return Err(format!(
+ "override for '{w_id}' must be a non-negative integer number of seconds, got {v}"
+ ));
+ }
+ }
+ }
+ Ok(overrides)
+}
+
+/// Loads the per-workspace retention overrides from the `retention_period_secs_overrides` global
+/// setting (a JSON `{workspace_id: secs}` object) into the in-memory `JOB_RETENTION_SECS_OVERRIDES`
+/// cache, so the cleanup sweep reads them without a per-tick DB query. Enterprise-only — CE leaves
+/// the cache empty so the sweep behaves exactly as before.
+///
+/// On a load error, unexpected value shape, or malformed data the previous map is kept but
+/// `JOB_RETENTION_SECS_OVERRIDES_LOADED` is set to FALSE, marking the cache unknown. Clobbering the
+/// map to empty would let the global sweep delete jobs a workspace asked to keep longer; leaving the
+/// flag TRUE would keep the stale (possibly shorter) policy in force after a lengthened/added
+/// override fails to refresh, deleting those jobs prematurely. Marking it unknown makes the sweep
+/// fail closed — it skips and the monitor retries the load next tick until a confirmed-current state
+/// loads. `LOADED` is set true only on a valid map, explicit unset (`Ok(None)`), or CE's no-op.
+pub async fn load_retention_period_overrides(db: &DB) -> error::Result<()> {
+ #[cfg(not(feature = "enterprise"))]
+ {
+ let _ = db;
+ // Overrides are EE-only; empty is the correct, fully-known state on CE.
+ JOB_RETENTION_SECS_OVERRIDES_LOADED.store(true, std::sync::atomic::Ordering::Relaxed);
+ }
+ #[cfg(feature = "enterprise")]
+ {
+ use windmill_common::global_settings::RETENTION_PERIOD_SECS_OVERRIDES_SETTING;
+ let value =
+ load_value_from_global_settings(db, RETENTION_PERIOD_SECS_OVERRIDES_SETTING).await;
+ match value {
+ Ok(Some(serde_json::Value::Object(map))) => match parse_retention_overrides(map) {
+ Ok(overrides) => {
+ JOB_RETENTION_SECS_OVERRIDES.store(std::sync::Arc::new(overrides));
+ JOB_RETENTION_SECS_OVERRIDES_LOADED
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ }
+ // Malformed persisted value: we can't confirm the current override set. Keep the
+ // last-good map but mark the cache unknown so the sweep fails closed (skips) and
+ // retries, rather than deleting with a stale — possibly shorter — policy.
+ Err(reason) => {
+ JOB_RETENTION_SECS_OVERRIDES_LOADED
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ tracing::error!(
+ "Malformed per-workspace retention overrides, gating cleanup until it loads: {reason}"
+ );
+ }
+ },
+ Ok(None) => {
+ // Explicit unset is a known state: no overrides.
+ JOB_RETENTION_SECS_OVERRIDES
+ .store(std::sync::Arc::new(std::collections::HashMap::new()));
+ JOB_RETENTION_SECS_OVERRIDES_LOADED
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ }
+ // Unexpected shape / read failure: mark unknown so a lengthened or added override that
+ // failed to refresh can't be missed by a sweep still running the previous policy.
+ Ok(Some(other)) => {
+ JOB_RETENTION_SECS_OVERRIDES_LOADED
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ tracing::error!(
+ "Per-workspace retention overrides setting is not a JSON object (got {other}); gating cleanup until it loads"
+ );
+ }
+ Err(e) => {
+ JOB_RETENTION_SECS_OVERRIDES_LOADED
+ .store(false, std::sync::atomic::Ordering::Relaxed);
+ tracing::error!(
+ "Error loading per-workspace retention overrides, gating cleanup until it loads: {e:#}"
+ );
+ }
+ }
+ }
+ Ok(())
+}
+
async fn delete_log_files_from_disk_and_store(
paths_to_delete: Vec,
tmp_dir: &str,
@@ -4734,3 +5013,54 @@ async fn manage_audit_partitions(db: &DB, retention_days: i64) {
}
}
}
+
+#[cfg(all(test, feature = "enterprise"))]
+mod retention_overrides_tests {
+ use super::parse_retention_overrides;
+ use serde_json::json;
+
+ fn obj(v: serde_json::Value) -> serde_json::Map {
+ v.as_object().unwrap().clone()
+ }
+
+ #[test]
+ fn parses_valid_map() {
+ let m = parse_retention_overrides(obj(json!({"a": 3600, "b": 0}))).unwrap();
+ assert_eq!(m.get("a"), Some(&3600));
+ assert_eq!(m.get("b"), Some(&0)); // 0 = keep forever, allowed
+ assert_eq!(m.len(), 2);
+ }
+
+ #[test]
+ fn empty_map_is_ok() {
+ assert!(parse_retention_overrides(obj(json!({})))
+ .unwrap()
+ .is_empty());
+ }
+
+ #[test]
+ fn rejects_negative() {
+ // A negative value must not silently become keep-forever; the whole map is rejected.
+ assert!(parse_retention_overrides(obj(json!({"a": 3600, "b": -1}))).is_err());
+ }
+
+ #[test]
+ fn rejects_non_integer() {
+ assert!(parse_retention_overrides(obj(json!({"a": "3600"}))).is_err());
+ assert!(parse_retention_overrides(obj(json!({"a": 3600.5}))).is_err());
+ assert!(parse_retention_overrides(obj(json!({"a": null}))).is_err());
+ }
+
+ #[test]
+ fn rejects_too_many_overrides() {
+ use windmill_common::global_settings::MAX_RETENTION_OVERRIDE_WORKSPACES;
+ let at_cap: serde_json::Map<_, _> = (0..MAX_RETENTION_OVERRIDE_WORKSPACES)
+ .map(|i| (format!("ws_{i}"), json!(3600)))
+ .collect();
+ assert!(parse_retention_overrides(at_cap.clone()).is_ok());
+ let over_cap: serde_json::Map<_, _> = (0..MAX_RETENTION_OVERRIDE_WORKSPACES + 1)
+ .map(|i| (format!("ws_{i}"), json!(3600)))
+ .collect();
+ assert!(parse_retention_overrides(over_cap).is_err());
+ }
+}
diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs
index 6ec3f9e410..791556b035 100644
--- a/backend/tests/agent_workers.rs
+++ b/backend/tests/agent_workers.rs
@@ -22,7 +22,7 @@ fn bun_code(code: &str) -> RawCode {
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
modules: None,
- tag: None,
+ tag: None,
}
}
diff --git a/backend/tests/app_s3_onbehalf.rs b/backend/tests/app_s3_onbehalf.rs
new file mode 100644
index 0000000000..2e1085edd3
--- /dev/null
+++ b/backend/tests/app_s3_onbehalf.rs
@@ -0,0 +1,521 @@
+//! Deployed-app S3 reads authorize on-behalf of the app author and are confined
+//! to app provenance (declared keys or recent job outputs): a viewer cannot read
+//! an arbitrary `file_key` as the author. Requires the `parquet` feature — the
+//! real `apps_u/*` S3 handlers are gated on it.
+//!
+//! `base` fixture: test-user (admin, SECRET_TOKEN); test-user-2 (non-admin,
+//! SECRET_TOKEN_2, no S3 folder permission).
+#![cfg(feature = "parquet")]
+
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+const ADMIN_TOKEN: &str = "SECRET_TOKEN";
+const USER_TOKEN: &str = "SECRET_TOKEN_2";
+const APP: &str = "u/test-user/s3onbehalf";
+const DECLARED: &str = "provenance/allowed.csv";
+const NON_PROVENANCE: &str = "evil/secret.csv";
+
+fn client() -> reqwest::Client {
+ reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder {
+ builder.header("Authorization", format!("Bearer {}", token))
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+ // `on_behalf_of` is auto-set to the creator (admin) for an anonymous app, so
+ // the app reads S3 as that author; `DECLARED` is the only allowlisted key.
+ let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN)
+ .json(&json!({
+ "path": APP,
+ "summary": "s3 onbehalf test",
+ "value": {},
+ "policy": {
+ "execution_mode": "anonymous",
+ "triggerables": {},
+ "allowed_s3_keys": [{ "s3_path": DECLARED }]
+ }
+ }))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
+
+ // GET an app-scoped S3 route as `token`. No workspace storage is configured,
+ // so a request that clears the provenance gate fails later at the storage
+ // lookup (or the CE OSS stub), never with "File restricted" — which is what
+ // lets these assertions distinguish "gate passed" from "gate denied".
+ let get = |route: &str, token: &'static str| {
+ let url = format!("{ws}/apps_u/{route}");
+ authed(client().get(url), token).send()
+ };
+ let denied = |body: &str| body.contains("File restricted");
+
+ // download_s3_file: author-on-behalf allowed for the declared key, denied for
+ // a key the app never declared (the confused-deputy guard).
+ let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}"), USER_TOKEN)
+ .await?
+ .text()
+ .await?;
+ assert!(!denied(&body), "declared key must clear the gate: {body}");
+ let body = get(
+ &format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"),
+ USER_TOKEN,
+ )
+ .await?
+ .text()
+ .await?;
+ assert!(denied(&body), "non-provenance key must be denied: {body}");
+
+ // load_table_count and load_csv_preview enforce the same gate. The preview's
+ // numeric `limit`/`offset` must deserialize (regression: a flattened query
+ // struct 400s on them under serde_urlencoded).
+ let body = get(
+ &format!("load_table_count/{APP}?file_key={DECLARED}"),
+ USER_TOKEN,
+ )
+ .await?
+ .text()
+ .await?;
+ assert!(
+ !denied(&body),
+ "table_count declared key must clear the gate: {body}"
+ );
+ let body = get(
+ &format!("load_table_count/{APP}?file_key={NON_PROVENANCE}"),
+ USER_TOKEN,
+ )
+ .await?
+ .text()
+ .await?;
+ assert!(
+ denied(&body),
+ "table_count non-provenance key must be denied: {body}"
+ );
+
+ let resp = get(
+ &format!("load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0"),
+ USER_TOKEN,
+ )
+ .await?;
+ let status = resp.status();
+ let body = resp.text().await?;
+ assert_ne!(status, 400, "numeric limit/offset must deserialize: {body}");
+ assert!(
+ !denied(&body),
+ "csv_preview declared key must clear the gate: {body}"
+ );
+
+ // load_file_preview: `read_bytes_from` / `read_bytes_length` are required.
+ let resp = get(
+ &format!("load_file_preview/{APP}?file_key={DECLARED}"),
+ USER_TOKEN,
+ )
+ .await?;
+ assert_eq!(
+ resp.status(),
+ 400,
+ "file_preview without byte range must 400: {}",
+ resp.text().await?
+ );
+ let body = get(
+ &format!(
+ "load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096"
+ ),
+ USER_TOKEN,
+ )
+ .await?
+ .text()
+ .await?;
+ assert!(
+ !denied(&body),
+ "file_preview declared key must clear the gate: {body}"
+ );
+
+ Ok(())
+}
+
+/// Seed a completed job whose result carries an s3 object. `app_trigger` sets the
+/// app-origination marker exactly as `execute_component` stamps it: `Some(app_path)`
+/// => `trigger_kind = 'app'` + `trigger = ` (an app-launched run);
+/// `None` => an ordinary direct `/jobs/run` (no app marker). `created_by` is the user
+/// the job ran as (the isolation key the gate confines downloads to).
+async fn seed_completed_job(
+ db: &Pool,
+ created_by: &str,
+ app_trigger: Option<&str>,
+ s3_key: &str,
+) -> anyhow::Result<()> {
+ let result = format!(r#"{{"s3":"{s3_key}"}}"#);
+ sqlx::query(
+ r#"
+ WITH j AS (
+ INSERT INTO v2_job (id, workspace_id, kind, runnable_path, created_by,
+ permissioned_as, trigger_kind, trigger)
+ VALUES (gen_random_uuid(), 'test-workspace', 'script', 'u/test-user/query_to_s3',
+ $1, 'u/test-user',
+ CASE WHEN $2::text IS NULL THEN NULL ELSE 'app'::job_trigger_kind END, $2)
+ RETURNING id
+ )
+ INSERT INTO v2_job_completed (id, workspace_id, duration_ms, status, result, started_at)
+ SELECT id, 'test-workspace', 1, 'success', $3::jsonb, now() FROM j
+ "#,
+ )
+ .bind(created_by)
+ .bind(app_trigger)
+ .bind(&result)
+ .execute(db)
+ .await?;
+ Ok(())
+}
+
+/// A deployed app that renders S3 files it produced (e.g. a SQL query persisted to
+/// S3 by a component) must clear the provenance gate for the viewer whose own app
+/// run produced them, while (a) a viewer cannot forge provenance by running a
+/// runnable directly (no app marker), (b) another app's outputs stay denied, and
+/// (c) another viewer's outputs stay denied (cross-viewer isolation). Provenance is
+/// keyed on the app-origination marker (`trigger_kind='app'` + `trigger=`)
+/// that `execute_component` stamps, plus `created_by = ` for isolation.
+#[sqlx::test(fixtures("base"))]
+async fn test_deployed_app_s3_onbehalf_flow_script_provenance(
+ db: Pool,
+) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+ const FS_APP: &str = "u/test-user/s3flowscript";
+ const OTHER_APP: &str = "u/test-user/other_app";
+ // Produced by test-user-2's own app run of THIS app.
+ const USER_KEY: &str = "results/user2_output.parquet";
+ // Produced by test-user's own app run of THIS app.
+ const ADMIN_KEY: &str = "results/admin_output.parquet";
+ // Produced by an app run of a DIFFERENT app → must stay denied.
+ const OTHER_APP_KEY: &str = "results/other_app_output.parquet";
+ // Produced by a DIRECT run (no app marker) → the forgery attempt, must stay denied.
+ const FORGED_KEY: &str = "results/author_only_secret.parquet";
+
+ let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN)
+ .json(&json!({
+ "path": FS_APP,
+ "summary": "s3 app-origination provenance test",
+ "value": {},
+ "policy": { "execution_mode": "anonymous", "triggerables": {} }
+ }))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
+
+ // Seed the produced-file jobs (all within the 3h window).
+ seed_completed_job(&db, "test-user-2", Some(FS_APP), USER_KEY).await?;
+ seed_completed_job(&db, "test-user", Some(FS_APP), ADMIN_KEY).await?;
+ seed_completed_job(&db, "test-user-2", Some(OTHER_APP), OTHER_APP_KEY).await?;
+ seed_completed_job(&db, "test-user-2", None, FORGED_KEY).await?;
+
+ let get = |route: &str, token: &'static str| {
+ let url = format!("{ws}/apps_u/{route}");
+ authed(client().get(url), token).send()
+ };
+ let denied = |body: &str| body.contains("File restricted");
+ let body_of = |route: String, token: &'static str| async move {
+ get(&route, token).await.unwrap().text().await.unwrap()
+ };
+
+ // The viewer's own app run's output clears the gate (the case that regressed to
+ // "File restricted").
+ let body = body_of(
+ format!("download_s3_file/{FS_APP}?s3={USER_KEY}"),
+ USER_TOKEN,
+ )
+ .await;
+ assert!(
+ !denied(&body),
+ "viewer's own app-produced key must clear the gate: {body}"
+ );
+
+ // The admin viewer's own app run's output clears — the gate has no admin bypass,
+ // it just matches the caller's own runs.
+ let body = body_of(
+ format!("download_s3_file/{FS_APP}?s3={ADMIN_KEY}"),
+ ADMIN_TOKEN,
+ )
+ .await;
+ assert!(
+ !denied(&body),
+ "admin's own app-produced key must clear the gate: {body}"
+ );
+
+ // Cross-viewer isolation: the admin cannot pull test-user-2's result even though
+ // it is a genuine app-marked job of the same app (no admin bypass either).
+ let body = body_of(
+ format!("download_s3_file/{FS_APP}?s3={USER_KEY}"),
+ ADMIN_TOKEN,
+ )
+ .await;
+ assert!(
+ denied(&body),
+ "another viewer's app-produced key must stay denied (isolation): {body}"
+ );
+
+ // A key produced by a direct run (no app marker) stays denied — the forgery the
+ // app-origination marker closes.
+ let body = body_of(
+ format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}"),
+ USER_TOKEN,
+ )
+ .await;
+ assert!(
+ denied(&body),
+ "key from a direct run (no app marker) must stay denied: {body}"
+ );
+
+ // A key produced by a DIFFERENT app stays denied — provenance is scoped to THIS
+ // app's path.
+ let body = body_of(
+ format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}"),
+ USER_TOKEN,
+ )
+ .await;
+ assert!(
+ denied(&body),
+ "key produced by a different app must stay denied: {body}"
+ );
+
+ Ok(())
+}
+
+/// Seed a minimal deployed script so `execute_component` can resolve `script/`.
+/// The script is given its OWN `on_behalf_of` (created_by test-user-2), distinct from
+/// any app author, so a test can assert an app component runs as the app's identity,
+/// not the referenced script's on_behalf.
+async fn seed_script(db: &Pool, path: &str, content: &str) -> anyhow::Result<()> {
+ let mut h = 0i64;
+ for b in path.bytes().chain(content.bytes()) {
+ h = h.wrapping_mul(31).wrapping_add(b as i64);
+ }
+ sqlx::query(
+ r#"INSERT INTO script (workspace_id, hash, path, summary, description, content,
+ created_by, on_behalf_of_email, language, tag, lock)
+ VALUES ('test-workspace', $1, $2, '', '', $3, 'test-user-2', 'test2@windmill.dev',
+ 'deno'::script_lang, 'deno', '')
+ ON CONFLICT DO NOTHING"#,
+ )
+ .bind(h)
+ .bind(path)
+ .bind(content)
+ .execute(db)
+ .await?;
+ // #[sqlx::test] isolated DBs share one workspace id and reuse script paths; the
+ // process-global deployed-script cache is keyed by (workspace, path), so disable
+ // it here so `execute_component` resolves against this test's own DB.
+ windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED
+ .store(true, std::sync::atomic::Ordering::Relaxed);
+ Ok(())
+}
+
+/// End-to-end: `execute_component` must stamp the job it enqueues with
+/// `trigger_kind = 'app'` + `trigger = `. This is the marker the S3
+/// provenance gate relies on; the gate tests seed it directly, so this test proves
+/// the runtime actually produces it. `execute_component` commits the job row and
+/// returns its id, so we assert on the row without needing a worker to run it.
+#[sqlx::test(fixtures("base"))]
+async fn test_execute_component_stamps_app_trigger(db: Pool) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+ const APP_PATH: &str = "u/test-user/trigger_marker_app";
+ const SCRIPT_PATH: &str = "u/test-user/query_to_s3";
+
+ seed_script(&db, SCRIPT_PATH, "export function main() { return 1 }").await?;
+
+ // Anonymous app wired to run the deployed script; keys use the production
+ // component-prefixed triggerable form.
+ let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN)
+ .json(&json!({
+ "path": APP_PATH,
+ "summary": "trigger marker test",
+ "value": {},
+ "policy": {
+ "execution_mode": "anonymous",
+ "triggerables_v2": {
+ format!("comp1:script/{SCRIPT_PATH}"): { "static_inputs": {}, "one_of_inputs": {} }
+ }
+ }
+ }))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
+
+ // Run the script component through the app runtime.
+ let resp = authed(
+ client().post(format!("{ws}/apps_u/execute_component/{APP_PATH}")),
+ ADMIN_TOKEN,
+ )
+ .json(&json!({
+ "component": "comp1",
+ "path": format!("script/{SCRIPT_PATH}"),
+ "args": {}
+ }))
+ .send()
+ .await?;
+ let status = resp.status();
+ let job_id = resp.text().await?;
+ assert_eq!(status, 200, "execute_component: {job_id}");
+ let job_id = job_id.trim().trim_matches('"');
+
+ // The enqueued job must carry the app-origination marker (trigger_kind = 'app',
+ // trigger = the app path, NOT the runnable path) and must run on-behalf of the
+ // APP's identity (u/test-user, the anonymous app's author), NOT the referenced
+ // script's own on_behalf (u/test-user-2).
+ let (trigger_kind, trigger, permissioned_as): (Option, Option, String) =
+ sqlx::query_as(
+ "SELECT trigger_kind::text, trigger, permissioned_as FROM v2_job \
+ WHERE id = $1::uuid AND workspace_id = 'test-workspace'",
+ )
+ .bind(job_id)
+ .fetch_one(&db)
+ .await?;
+
+ assert_eq!(
+ trigger_kind.as_deref(),
+ Some("app"),
+ "execute_component must stamp trigger_kind = 'app' (got {trigger_kind:?})"
+ );
+ assert_eq!(
+ trigger.as_deref(),
+ Some(APP_PATH),
+ "trigger must be the app path, not the runnable path (got {trigger:?})"
+ );
+ assert_eq!(
+ permissioned_as, "u/test-user",
+ "component must run on-behalf of the APP identity, not the referenced script's on_behalf (got {permissioned_as})"
+ );
+
+ Ok(())
+}
+
+/// `JobTriggerKind::App` (added for the app-origination S3 marker) is now a valid
+/// value for the suspended-trigger reassignment routes, but there is no
+/// `app_trigger` table. The handler must reject it with a clean 400 rather than
+/// failing on a missing-relation database error (500).
+#[sqlx::test(fixtures("base"))]
+async fn test_app_trigger_kind_rejected_for_reassignment(db: Pool) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+ let resp = authed(
+ client().post(format!(
+ "{ws}/trigger/app/resume_suspended_trigger_jobs/u/test-user/x"
+ )),
+ ADMIN_TOKEN,
+ )
+ .json(&json!({}))
+ .send()
+ .await?;
+ let status = resp.status();
+ let body = resp.text().await?;
+ assert_eq!(
+ status, 400,
+ "app reassignment must be a clean 400, not 500: {body}"
+ );
+ assert!(
+ body.contains("do not support job reassignment"),
+ "expected reassignment-unsupported message, got: {body}"
+ );
+
+ Ok(())
+}
+
+/// A preview run is NEVER app-provenanced. Preview executes as the *caller* (Viewer
+/// mode), so its results are read back as the caller via the viewer-scoped
+/// job_helpers endpoint — never author-mode. Marking a preview would let any
+/// `jobs:run` caller supply arbitrary `raw_code` against a victim app path and forge
+/// the marker the S3 gate trusts; and it is never needed. Even the app owner's own
+/// preview stays unmarked.
+#[sqlx::test(fixtures("base"))]
+async fn test_preview_is_not_app_provenanced(db: Pool) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let ws = format!("http://localhost:{port}/api/w/test-workspace");
+
+ const APP: &str = "u/test-user/preview_app";
+ let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN)
+ .json(&json!({
+ "path": APP,
+ "summary": "preview marker test",
+ "value": {},
+ "policy": { "execution_mode": "anonymous", "triggerables": {} }
+ }))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
+
+ // Preview arbitrary inline code against the app (force_viewer_static_fields =>
+ // preview mode; raw_code with no path/id skips all app authorization), as `token`.
+ let preview_trigger_kind = |token: &'static str| {
+ let ws = ws.clone();
+ let db = db.clone();
+ async move {
+ let resp = authed(
+ client().post(format!("{ws}/apps_u/execute_component/{APP}")),
+ token,
+ )
+ .json(&json!({
+ "component": "comp1",
+ "raw_code": { "content": "export function main() { return 1 }", "language": "deno" },
+ "force_viewer_static_fields": {},
+ "args": {}
+ }))
+ .send()
+ .await
+ .unwrap();
+ let status = resp.status();
+ let job_id = resp.text().await.unwrap();
+ assert_eq!(status, 200, "preview execute_component: {job_id}");
+ let job_id = job_id.trim().trim_matches('"').to_string();
+ let trigger_kind: Option = sqlx::query_scalar(
+ "SELECT trigger_kind::text FROM v2_job WHERE id = $1::uuid AND workspace_id = 'test-workspace'",
+ )
+ .bind(job_id)
+ .fetch_one(&db)
+ .await
+ .unwrap();
+ trigger_kind
+ }
+ };
+
+ // The app owner (test-user, admin) previewing their own app → still NOT marked.
+ let trigger_kind = preview_trigger_kind(ADMIN_TOKEN).await;
+ assert_eq!(
+ trigger_kind, None,
+ "the app owner's own preview must NOT be app-provenanced (got {trigger_kind:?})"
+ );
+
+ // A non-editor (test-user-2) previewing a victim app → NOT marked.
+ let trigger_kind = preview_trigger_kind(USER_TOKEN).await;
+ assert_eq!(
+ trigger_kind, None,
+ "a non-editor's preview must NOT be app-provenanced (got {trigger_kind:?})"
+ );
+
+ Ok(())
+}
diff --git a/backend/tests/sign_s3_objects_authz.rs b/backend/tests/sign_s3_objects_authz.rs
new file mode 100644
index 0000000000..9acf4352c3
--- /dev/null
+++ b/backend/tests/sign_s3_objects_authz.rs
@@ -0,0 +1,124 @@
+//! Regression test for the `sign_s3_objects` permission bypass.
+//!
+//! Invariant: minting an S3 read signature (`apps/sign_s3_objects`) requires the
+//! CALLER to hold `S3Permission::READ` on the key. The signature is a transferable
+//! bearer capability (`validate_s3_signature` only checks HMAC + expiry), so a
+//! caller must not be able to sign a key they cannot themselves read — otherwise
+//! any workspace member could bypass the advanced S3 permission rules.
+//!
+//! Pinned against a FilesystemStorage LFS whose advanced permissions grant a
+//! non-admin READ on `allowed/*` but nothing on `secret/*`:
+//! - the non-admin CAN sign `allowed/*` (authorized), and the minted signature
+//! validates end-to-end through the presigned s3_proxy fetch route;
+//! - the non-admin CANNOT sign `secret/*` (bypass closed);
+//! Advanced S3 permissions are an enterprise feature, so this test requires the
+//! `enterprise` + `private` + `parquet` features.
+#![cfg(all(feature = "enterprise", feature = "private", feature = "parquet"))]
+
+use serde_json::json;
+use sqlx::{Pool, Postgres};
+use windmill_test_utils::*;
+
+fn client() -> reqwest::Client {
+ reqwest::Client::new()
+}
+
+fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder {
+ builder.header("Authorization", format!("Bearer {}", token))
+}
+
+/// Configure the workspace LFS as a filesystem store rooted at `root_path`, with
+/// an advanced permission rule granting READ on `allowed/*` to everyone the glob
+/// matches (non-admins included). No rule covers `secret/*`, so it is denied.
+async fn configure_lfs(db: &Pool, root_path: &str) -> anyhow::Result<()> {
+ let lfs_config = json!({
+ "type": "FilesystemStorage",
+ "root_path": root_path,
+ "public_resource": null,
+ "advanced_permissions": [
+ { "pattern": "allowed/*", "allow": "read" }
+ ]
+ });
+ sqlx::query!(
+ "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2",
+ lfs_config,
+ "test-workspace"
+ )
+ .execute(db)
+ .await?;
+ Ok(())
+}
+
+#[sqlx::test(fixtures("base"))]
+async fn test_sign_s3_objects_enforces_read_authz(db: Pool) -> anyhow::Result<()> {
+ initialize_tracing().await;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+ let base = format!("http://localhost:{port}/api/w/test-workspace");
+
+ let storage_dir = tempfile::tempdir()?;
+ let storage_root = storage_dir.path().to_string_lossy().to_string();
+ configure_lfs(&db, &storage_root).await?;
+
+ // A real object so the signed fetch can stream bytes end-to-end.
+ let allowed_dir = storage_dir.path().join("allowed");
+ std::fs::create_dir_all(&allowed_dir)?;
+ std::fs::write(allowed_dir.join("file.txt"), b"authorized payload")?;
+
+ // ---- CORE REGRESSION: a non-admin (test-user-2) may NOT sign a key they have
+ // no READ permission on. Before the fix this returned a valid signature.
+ let resp = authed(
+ client().post(format!("{base}/apps/sign_s3_objects")),
+ "SECRET_TOKEN_2",
+ )
+ .json(&json!({ "s3_objects": [{ "s3": "secret/file.txt" }] }))
+ .send()
+ .await?;
+ let status = resp.status();
+ let body = resp.text().await?;
+ assert!(
+ !status.is_success(),
+ "non-admin must NOT be able to sign a key they cannot read (bypass): {status} {body}"
+ );
+
+ // ---- NO OVER-BLOCKING: the same non-admin CAN sign a key their advanced
+ // permissions allow them to read.
+ let resp = authed(
+ client().post(format!("{base}/apps/sign_s3_objects")),
+ "SECRET_TOKEN_2",
+ )
+ .json(&json!({ "s3_objects": [{ "s3": "allowed/file.txt" }] }))
+ .send()
+ .await?;
+ let status = resp.status();
+ let signed: serde_json::Value = resp.json().await?;
+ assert!(
+ status.is_success(),
+ "non-admin must be able to sign a key they can read: {status} {signed}"
+ );
+ let presigned = signed[0]["presigned"]
+ .as_str()
+ .expect("authorized sign must return a presigned string")
+ .to_string();
+
+ // ---- END-TO-END: the minted signature is accepted by the fetch-side gate.
+ // Hit the presigned s3_proxy route (default storage) and confirm it
+ // streams the object rather than rejecting the signature.
+ let fetch_url = format!("{base}/s3_proxy/_default_/allowed/file.txt?{presigned}");
+ let resp = client().get(&fetch_url).send().await?;
+ let status = resp.status();
+ let body = resp.bytes().await?;
+ assert!(
+ status.is_success(),
+ "signed fetch of an authorized key must succeed end-to-end: {status} {:?}",
+ String::from_utf8_lossy(&body)
+ );
+ assert_eq!(
+ body.as_ref(),
+ b"authorized payload",
+ "signed fetch must stream the authorized object's bytes"
+ );
+
+ Ok(())
+}
diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs
index b1bf2d4273..25f572d8ae 100644
--- a/backend/windmill-api-auth/src/lib.rs
+++ b/backend/windmill-api-auth/src/lib.rs
@@ -541,6 +541,16 @@ pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> {
}
if !path.is_empty() {
let splitted = path.split("/").collect::>();
+ // A valid path is at least `/` (e.g. `u/alice/...`,
+ // `f/folder/...`). Guard the `splitted[1]` accesses below so a
+ // malformed single-segment path returns a clear error instead of
+ // panicking with an out-of-bounds index.
+ if splitted.len() < 2 {
+ return Err(Error::BadRequest(format!(
+ "Invalid path '{}': a valid path starts with 'u//' or 'f//'",
+ path
+ )));
+ }
if splitted[0] == "u" {
if splitted[1] == authed.username {
Ok(())
@@ -1131,6 +1141,26 @@ mod tests {
);
}
+ // Regression for WIN-2157: a malformed single-segment path (e.g. a draft
+ // saved at a bare `u`) must return a clear error, not panic on the
+ // `splitted[1]` index. Non-admins reach this branch (admins short-circuit).
+ #[test]
+ fn require_owner_of_path_rejects_malformed_path_without_panicking() {
+ let alice = ApiAuthed { username: "alice".into(), ..Default::default() };
+ for path in ["u", "f", "g", "nonsense"] {
+ let err =
+ require_owner_of_path(&alice, path).expect_err("malformed path must be rejected");
+ assert!(
+ matches!(err, Error::BadRequest(_)),
+ "expected BadRequest for '{path}', got {err:?}"
+ );
+ }
+ // A well-formed foreign path returns the owner error, not a malformed one.
+ assert!(require_owner_of_path(&alice, "u/bob/script").is_err());
+ // The user's own namespace resolves.
+ assert!(require_owner_of_path(&alice, "u/alice/script").is_ok());
+ }
+
#[test]
fn predicate_no_scopes_allows_all() {
let authed = authed_with_scopes(None);
diff --git a/backend/windmill-api-integration-tests/tests/fork_secret_replication_azure.rs b/backend/windmill-api-integration-tests/tests/fork_secret_replication_azure.rs
new file mode 100644
index 0000000000..19682de70a
--- /dev/null
+++ b/backend/windmill-api-integration-tests/tests/fork_secret_replication_azure.rs
@@ -0,0 +1,161 @@
+//! End-to-end regression test for WIN-2161.
+//!
+//! Reproduces, through real product code, the state after a database-to-external
+//! migration: a secret that was created under the database backend and then
+//! *migrated* to an external backend (Azure Key Vault). Migration writes the
+//! plaintext to the store but
+//! leaves the encrypted ciphertext in `variable.value` (it never rewrites it to
+//! a `$azure_kv:` marker). The bug: `clone_variables` only replicated
+//! marker-valued secrets, so forking left the migrated secret unreplicated and
+//! reads in the fork failed with "not found in Azure Key Vault".
+//!
+//! This drives the real `/migrate_secrets_to_azure_kv`, `/create_fork` and
+//! `variables/get_value` endpoints against a local Azure Key Vault emulator
+//! (lowkey-vault), which the `AzureKeyVaultBackend` talks to via its
+//! static-token / self-signed-cert emulator mode.
+//!
+//! Run it:
+//! ```bash
+//! podman run -d --name lowkey -p 8443:8443 \
+//! -e LOWKEY_ARGS="--LOWKEY_VAULT_NAMES=default" \
+//! docker.io/nagyesta/lowkey-vault:7.3.0
+//!
+//! RUN_AZURE_KV_TESTS=1 cargo test -p windmill-api-integration-tests \
+//! --features private,enterprise --test fork_secret_replication_azure -- --nocapture
+//! ```
+
+#[cfg(all(feature = "private", feature = "enterprise"))]
+mod azure_fork {
+ use serde_json::json;
+ use sqlx::{Pool, Postgres};
+ use windmill_common::variables::{build_crypt, encrypt};
+ use windmill_test_utils::*;
+
+ fn client() -> reqwest::Client {
+ reqwest::Client::new()
+ }
+
+ fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+ builder.header("Authorization", "Bearer SECRET_TOKEN")
+ }
+
+ fn vault_url() -> String {
+ std::env::var("AZURE_KV_URL").unwrap_or_else(|_| "https://localhost:8443".to_string())
+ }
+
+ /// The Azure settings for the emulator: a static token switches the backend
+ /// into emulator mode (no Entra ID, self-signed certs accepted).
+ fn azure_settings() -> serde_json::Value {
+ json!({
+ "vault_url": vault_url(),
+ "tenant_id": "emulator-tenant",
+ "client_id": "emulator-client",
+ "token": "emulator-token",
+ })
+ }
+
+ #[sqlx::test(migrations = "../migrations", fixtures("base"))]
+ async fn migrated_secret_is_replicated_on_fork(db: Pool) -> anyhow::Result<()> {
+ if std::env::var("RUN_AZURE_KV_TESTS").as_deref() != Ok("1") {
+ eprintln!("skipping: set RUN_AZURE_KV_TESTS=1 and start lowkey-vault to run");
+ return Ok(());
+ }
+ initialize_tracing().await;
+
+ // The Azure KV emulator persists across runs; derive unique names per run
+ // so a secret written by a previous run can't mask a regression.
+ let suffix = uuid::Uuid::new_v4().simple().to_string();
+ let short = &suffix[..8];
+ let source_ws = "test-workspace";
+ let path = format!("u/test-user/db_password_{short}");
+ let path = path.as_str();
+ let plaintext = "s3cr3t-value";
+
+ let ciphertext = {
+ let mc = build_crypt(&db, source_ws).await?;
+ encrypt(&mc, plaintext)
+ };
+ sqlx::query(
+ "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
+ VALUES ($1, $2, $3, true, '', '{}')",
+ )
+ .bind(source_ws)
+ .bind(path)
+ .bind(&ciphertext)
+ .execute(&db)
+ .await?;
+
+ sqlx::query(
+ "INSERT INTO global_settings (name, value) VALUES ('secret_backend', $1)
+ ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value",
+ )
+ .bind(json!({
+ "type": "AzureKeyVault",
+ "vault_url": vault_url(),
+ "tenant_id": "emulator-tenant",
+ "client_id": "emulator-client",
+ "token": "emulator-token",
+ }))
+ .execute(&db)
+ .await?;
+
+ let server = ApiServer::start(db.clone()).await?;
+ let port = server.addr.port();
+
+ let resp = authed(client().post(format!(
+ "http://localhost:{port}/api/settings/migrate_secrets_to_azure_kv"
+ )))
+ .json(&azure_settings())
+ .send()
+ .await?;
+ let status = resp.status();
+ let body = resp.text().await?;
+ assert_eq!(status, 200, "migrate_secrets_to_azure_kv failed: {body}");
+ let report: serde_json::Value = serde_json::from_str(&body)?;
+ assert!(
+ report["migrated_count"].as_i64().unwrap_or(0) >= 1,
+ "expected at least one migrated secret: {report}"
+ );
+
+ // Assert the source resolves before forking, so a fork-read failure is
+ // attributable to replication rather than a broken seed.
+ let resp = authed(client().get(format!(
+ "http://localhost:{port}/api/w/{source_ws}/variables/get_value/{path}"
+ )))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 200, "source read: {}", resp.text().await?);
+ assert_eq!(resp.json::().await?, plaintext);
+
+ let fork_ws = format!("wm-fork-az{short}");
+ let fork_ws = fork_ws.as_str();
+ let resp = authed(client().post(format!(
+ "http://localhost:{port}/api/w/{source_ws}/workspaces/create_fork"
+ )))
+ .json(&json!({ "id": fork_ws, "name": "Azure Fork Test" }))
+ .send()
+ .await?;
+ assert_eq!(resp.status(), 200, "create_fork: {}", resp.text().await?);
+
+ // The fork resolves the secret only if it was replicated under the fork's
+ // own workspace-id key in the external store.
+ let resp = authed(client().get(format!(
+ "http://localhost:{port}/api/w/{fork_ws}/variables/get_value/{path}"
+ )))
+ .send()
+ .await?;
+ let status = resp.status();
+ let body = resp.text().await?;
+ assert_eq!(
+ status, 200,
+ "forked secret must resolve, got {status}: {body}"
+ );
+ assert_eq!(
+ serde_json::from_str::(&body)?,
+ plaintext,
+ "fork should return the replicated plaintext"
+ );
+
+ Ok(())
+ }
+}
diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs
index 8ec60a69ba..b4707fca9f 100644
--- a/backend/windmill-api-jobs/src/execution.rs
+++ b/backend/windmill-api-jobs/src/execution.rs
@@ -12,6 +12,7 @@ use axum::{
response::{IntoResponse, Response},
Json,
};
+use base64::Engine as _;
use http::{HeaderMap, HeaderName, HeaderValue};
use hyper::StatusCode;
use serde::Deserialize;
@@ -254,6 +255,8 @@ pub struct WindmillCompositeResult {
windmill_content_type: Option,
#[serde(alias = "wm_headers")]
windmill_headers: Option>,
+ #[serde(alias = "wm_content_transfer_encoding")]
+ windmill_content_transfer_encoding: Option,
result: Option>,
}
@@ -375,11 +378,13 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result
windmill_status_code,
windmill_content_type,
windmill_headers,
+ windmill_content_transfer_encoding,
result: result_value,
}) => {
if windmill_content_type.is_none()
&& windmill_status_code.is_none()
&& windmill_headers.is_none()
+ && windmill_content_transfer_encoding.is_none()
{
return Ok((
if success {
@@ -425,18 +430,54 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result
let serialized_json_result = result_value
.map(|val| val.get().to_owned())
.unwrap_or_else(String::new);
- let serialized_result =
- serde_json::from_str::(serialized_json_result.as_str())
- .ok()
- .unwrap_or(serialized_json_result);
+ let parsed_string =
+ serde_json::from_str::(serialized_json_result.as_str()).ok();
+ let result_is_json_string = parsed_string.is_some();
+ let serialized_result = parsed_string.unwrap_or(serialized_json_result);
headers.insert(
http::header::CONTENT_TYPE,
HeaderValue::from_str(content_type.as_str()).map_err(|err| {
Error::internal_err(format!("Invalid content type {content_type}: {err}"))
})?,
);
+ // Invalid base64 is a hard error, never a silent fallback to the encoded text.
+ match windmill_content_transfer_encoding.as_deref() {
+ Some("base64") => {
+ // Only a JSON string carries base64; a number/bool/null/array/object
+ // must not have its raw JSON text decoded into arbitrary bytes.
+ if !result_is_json_string {
+ return Err(Error::ExecutionErr(
+ "windmill_content_transfer_encoding \"base64\" requires result \
+ to be a base64-encoded string"
+ .to_string(),
+ ));
+ }
+ let decoded = base64::engine::general_purpose::STANDARD
+ .decode(serialized_result.as_bytes())
+ .map_err(|err| {
+ Error::ExecutionErr(format!(
+ "windmill_content_transfer_encoding is \"base64\" but the \
+ result is not valid base64: {err}"
+ ))
+ })?;
+ return Ok((status_code_or_default, headers, decoded).into_response());
+ }
+ Some(other) => {
+ return Err(Error::ExecutionErr(format!(
+ "Unsupported windmill_content_transfer_encoding \"{other}\" \
+ (only \"base64\" is supported)"
+ )));
+ }
+ None => {}
+ }
return Ok((status_code_or_default, headers, serialized_result).into_response());
}
+ if windmill_content_transfer_encoding.is_some() {
+ return Err(Error::ExecutionErr(
+ "windmill_content_transfer_encoding requires windmill_content_type to be set"
+ .to_string(),
+ ));
+ }
if let Some(result_value) = result_value {
return Ok((status_code_or_default, headers, Json(result_value)).into_response());
} else {
@@ -960,3 +1001,106 @@ pub async fn push_script_job_by_path_into_queue<'c>(
Ok((uuid, resolved_delete_secs, None))
}
}
+
+#[cfg(test)]
+mod result_to_response_tests {
+ use super::*;
+
+ fn raw(json: &str) -> Box {
+ serde_json::from_str(json).expect("valid json")
+ }
+
+ async fn body_bytes(resp: Response) -> Vec {
+ axum::body::to_bytes(resp.into_body(), usize::MAX)
+ .await
+ .expect("read body")
+ .to_vec()
+ }
+
+ #[tokio::test]
+ async fn base64_result_is_decoded_to_raw_bytes() {
+ // 0x00 0x01 0x02 0xFF is not valid UTF-8, so it can only survive as bytes.
+ let bytes = vec![0u8, 1, 2, 255];
+ let b64 = base64::engine::general_purpose::STANDARD.encode(&bytes);
+ let resp = result_to_response(
+ raw(&format!(
+ r#"{{"wm_content_type":"application/pdf","wm_content_transfer_encoding":"base64","result":"{b64}"}}"#
+ )),
+ true,
+ )
+ .expect("response");
+
+ assert_eq!(resp.status(), StatusCode::OK);
+ assert_eq!(
+ resp.headers().get(http::header::CONTENT_TYPE).unwrap(),
+ "application/pdf"
+ );
+ assert_eq!(body_bytes(resp).await, bytes);
+ }
+
+ #[tokio::test]
+ async fn invalid_base64_is_a_hard_error() {
+ let res = result_to_response(
+ raw(
+ r#"{"wm_content_type":"application/pdf","wm_content_transfer_encoding":"base64","result":"not valid base64!!"}"#,
+ ),
+ true,
+ );
+ assert!(res.is_err(), "invalid base64 must not silently fall back");
+ }
+
+ #[tokio::test]
+ async fn base64_mode_rejects_non_string_results() {
+ // A number/bool whose raw JSON text happens to be valid base64 (right length,
+ // base64 alphabet) must not be decoded into bytes — it must be a hard error.
+ for result in ["12345678", "true", "null", "[1,2,3]"] {
+ let res = result_to_response(
+ raw(&format!(
+ r#"{{"wm_content_type":"application/octet-stream","wm_content_transfer_encoding":"base64","result":{result}}}"#
+ )),
+ true,
+ );
+ assert!(
+ res.is_err(),
+ "base64 mode must reject non-string result: {result}"
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn unsupported_transfer_encoding_is_rejected() {
+ let res = result_to_response(
+ raw(
+ r#"{"wm_content_type":"text/plain","wm_content_transfer_encoding":"gzip","result":"x"}"#,
+ ),
+ true,
+ );
+ assert!(res.is_err());
+ }
+
+ #[tokio::test]
+ async fn transfer_encoding_without_content_type_is_rejected() {
+ let res = result_to_response(
+ raw(r#"{"wm_content_transfer_encoding":"base64","result":"aGk="}"#),
+ true,
+ );
+ assert!(res.is_err());
+ }
+
+ #[tokio::test]
+ async fn string_result_is_still_served_verbatim() {
+ // Regression: without a transfer encoding, a string result is sent as-is
+ // (quotes stripped), not base64-decoded.
+ let resp = result_to_response(
+ raw(r#"{"wm_content_type":"text/html","result":"hi
"}"#),
+ true,
+ )
+ .expect("response");
+
+ assert_eq!(
+ resp.headers().get(http::header::CONTENT_TYPE).unwrap(),
+ "text/html"
+ );
+ assert_eq!(body_bytes(resp).await, b"hi
");
+ }
+}
diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs
index 0a58f1f4b3..9d37233228 100644
--- a/backend/windmill-api-settings/src/lib.rs
+++ b/backend/windmill-api-settings/src/lib.rs
@@ -61,7 +61,8 @@ use windmill_common::{
AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING,
EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING,
- HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, RUFF_CONFIG_SETTING,
+ HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
+ RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RUFF_CONFIG_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
WS_BASE_URL_SETTING,
@@ -1047,6 +1048,38 @@ async fn run_setting_pre_write_hook(
}
}
}
+ RETENTION_PERIOD_SECS_OVERRIDES_SETTING => {
+ // Reject a malformed map at write time so it can never be persisted. A persisted bad
+ // value (negative or non-integer) would fail to parse on the next server start and,
+ // because the loader fails closed (skips cleanup until a known-good value is read),
+ // silently disable ALL job-retention cleanup indefinitely. This shape check must stay in
+ // sync with `parse_retention_overrides` in backend/src/monitor.rs.
+ match value {
+ // Clearing (delete row) is handled by the caller; allow it through.
+ serde_json::Value::Null => {}
+ serde_json::Value::String(s) if s.trim().is_empty() => {}
+ serde_json::Value::Object(map) => {
+ if map.len() > MAX_RETENTION_OVERRIDE_WORKSPACES {
+ return Err(error::Error::BadRequest(format!(
+ "retention_period_secs_overrides: at most {MAX_RETENTION_OVERRIDE_WORKSPACES} per-workspace overrides are allowed, got {}",
+ map.len()
+ )));
+ }
+ for (ws, v) in map {
+ if !v.as_i64().is_some_and(|secs| secs >= 0) {
+ return Err(error::Error::BadRequest(format!(
+ "retention_period_secs_overrides: override for '{ws}' must be a non-negative integer number of seconds, got {v}"
+ )));
+ }
+ }
+ }
+ _ => {
+ return Err(error::Error::BadRequest(
+ "retention_period_secs_overrides must be a JSON object of {workspace_id: seconds}".to_string(),
+ ));
+ }
+ }
+ }
_ => {}
}
Ok(())
diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs
index f5e5eab680..ecbc38fcb7 100644
--- a/backend/windmill-api-settings/src/log_cleanup.rs
+++ b/backend/windmill-api-settings/src/log_cleanup.rs
@@ -30,7 +30,10 @@ use windmill_common::error::{self};
use windmill_common::jobs::delete_jobs;
use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE};
use windmill_common::worker::WINDMILL_DIR;
-use windmill_common::{DB, INSTANCE_NAME, JOB_RETENTION_SECS, SERVICE_LOG_RETENTION_SECS};
+use windmill_common::{
+ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES,
+ JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS,
+};
use windmill_object_store::object_store_reexports::{
ObjectStore, ObjectStoreError, Path as ObjectPath,
@@ -321,29 +324,103 @@ async fn cleanup_job_logs(
store: &Arc,
) -> error::Result<()> {
let retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed);
- if retention_secs <= 0 {
+
+ // Per-workspace retention overrides (EE). Honor them exactly like the periodic monitor sweep:
+ // Phase 1 deletes on the instance window but EXCLUDES override workspaces, Phase 2 deletes each
+ // override workspace on its own window. Fail closed if the override set was never loaded (e.g.
+ // manual cleanup triggered right after startup) — sweeping globally with an unknown override set
+ // would delete jobs a longer-retention workspace asked to keep.
+ if !JOB_RETENTION_SECS_OVERRIDES_LOADED.load(std::sync::atomic::Ordering::Relaxed) {
+ tracing::warn!(
+ "log cleanup: per-workspace retention overrides not yet loaded; skipping job log cleanup this run"
+ );
return Ok(());
}
+ let overrides = JOB_RETENTION_SECS_OVERRIDES.load_full();
+ let override_ids: Vec = overrides.keys().cloned().collect();
+ let exclude: Option<&[String]> = if override_ids.is_empty() {
+ None
+ } else {
+ Some(&override_ids)
+ };
- let total: i64 = sqlx::query_scalar!(
- "SELECT COUNT(*) FROM v2_job_completed
- WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval",
- retention_secs,
- )
- .fetch_one(db)
- .await?
- .unwrap_or(0);
+ // Upfront total for the progress bar: Phase-1 candidates (instance window, excluding overrides)
+ // plus Phase-2 candidates (each override on its own window). Collapsed to `processed` at the end.
+ let mut total: i64 = if retention_secs > 0 {
+ sqlx::query_scalar!(
+ "SELECT COUNT(*) FROM v2_job_completed
+ WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND ($2::text[] IS NULL OR workspace_id NOT IN (
+ SELECT u FROM unnest($2::text[]) AS u WHERE u IS NOT NULL
+ ))",
+ retention_secs,
+ exclude,
+ )
+ .fetch_one(db)
+ .await?
+ .unwrap_or(0)
+ } else {
+ 0
+ };
+ for (w_id, secs) in overrides.iter() {
+ if *secs > 0 {
+ total += sqlx::query_scalar!(
+ "SELECT COUNT(*) FROM v2_job_completed
+ WHERE workspace_id = $1
+ AND completed_at <= now() - ($2::bigint::text || ' s')::interval",
+ w_id,
+ secs,
+ )
+ .fetch_one(db)
+ .await?
+ .unwrap_or(0);
+ }
+ }
session.update(|p| p.total_jobs = total as u64).await;
-
if total <= 0 {
return Ok(());
}
+ // Phase 1: instance window, excluding override workspaces.
+ if retention_secs > 0 {
+ run_job_log_cleanup_phase(session, db, store, retention_secs, None, exclude).await?;
+ }
+ // Phase 2: each override workspace on its own window (0 = keep forever, skipped).
+ for (w_id, secs) in overrides.iter() {
+ if *secs > 0 {
+ run_job_log_cleanup_phase(session, db, store, *secs, Some(w_id), None).await?;
+ }
+ }
+
+ // Collapse the total to what we actually processed — the upfront count includes jobs whose root
+ // is still active (protected from deletion), so without this the progress bar would get stuck.
+ session.update(|p| p.total_jobs = p.processed_jobs).await;
+
+ Ok(())
+}
+
+/// Runs the batched job+log delete loop for one retention scope (`only_workspace` / `exclude`),
+/// deleting the returned log blobs from storage and updating progress. See `cleanup_job_logs`.
+async fn run_job_log_cleanup_phase(
+ session: &Session,
+ db: &DB,
+ store: &Arc,
+ retention_secs: i64,
+ only_workspace: Option<&str>,
+ exclude_workspaces: Option<&[String]>,
+) -> error::Result<()> {
let mut completed_at_floor: Option> = None;
loop {
- let (deleted_count, rel_paths, max_completed_at) =
- delete_expired_jobs_batch(db, retention_secs, JOB_BATCH, completed_at_floor).await?;
+ let (deleted_count, rel_paths, max_completed_at) = delete_expired_jobs_batch(
+ db,
+ retention_secs,
+ JOB_BATCH,
+ completed_at_floor,
+ only_workspace,
+ exclude_workspaces,
+ )
+ .await?;
if deleted_count == 0 {
break;
@@ -369,12 +446,6 @@ async fn cleanup_job_logs(
})
.await;
}
-
- // Collapse the total to what we actually processed — the upfront count
- // includes jobs whose root is still active (protected from deletion), so
- // without this the progress bar would get stuck at e.g. 3/44.
- session.update(|p| p.total_jobs = p.processed_jobs).await;
-
Ok(())
}
@@ -386,6 +457,8 @@ async fn delete_expired_jobs_batch(
job_retention_secs: i64,
batch_size: i64,
completed_at_floor: Option>,
+ only_workspace: Option<&str>,
+ exclude_workspaces: Option<&[String]>,
) -> error::Result<(usize, Vec, Option>)> {
let mut tx = db.begin().await?;
@@ -401,53 +474,119 @@ async fn delete_expired_jobs_batch(
// `completed_at_floor` carries a watermark across batches so each one resumes after the rows
// the previous batch processed instead of re-scanning the (potentially undeletable) oldest
- // prefix; the empty-active-roots branch skips the v2_job join entirely. See
- // backend/src/monitor.rs::delete_expired_jobs_batch for the full rationale.
- let (deleted_jobs, max_completed_at) = if active_root_job_ids.is_empty() {
- let rows = sqlx::query!(
- "DELETE FROM v2_job_completed
- WHERE id IN (
- SELECT id FROM v2_job_completed
- WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
- AND ($3::timestamptz IS NULL OR completed_at >= $3)
- ORDER BY completed_at ASC
- LIMIT $2
- FOR UPDATE SKIP LOCKED
- )
- RETURNING id, completed_at",
- job_retention_secs,
- batch_size,
- completed_at_floor,
- )
- .fetch_all(&mut *tx)
- .await?;
- let max = rows.iter().map(|r| r.completed_at).max();
- (rows.into_iter().map(|r| r.id).collect::>(), max)
- } else {
- let rows = sqlx::query!(
- "DELETE FROM v2_job_completed
- WHERE id IN (
- SELECT jc.id FROM v2_job_completed jc
- LEFT JOIN v2_job j ON j.id = jc.id
- WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
- AND ($4::timestamptz IS NULL OR jc.completed_at >= $4)
- AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
- SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
- )
- ORDER BY jc.completed_at ASC
- LIMIT $2
- FOR UPDATE OF jc SKIP LOCKED
- )
- RETURNING id, completed_at",
- job_retention_secs,
- batch_size,
- &active_root_job_ids,
- completed_at_floor,
- )
- .fetch_all(&mut *tx)
- .await?;
- let max = rows.iter().map(|r| r.completed_at).max();
- (rows.into_iter().map(|r| r.id).collect::>(), max)
+ // prefix; the empty-active-roots branch skips the v2_job join entirely. Applied as
+ // `completed_at >= COALESCE($floor, '-infinity')` — the `$floor IS NULL OR ...` form is
+ // non-sargable and forces a Seq Scan. `only_workspace` / `exclude_workspaces` scope the sweep for
+ // the per-workspace retention override (Phase 1 global excluding override workspaces, Phase 2
+ // per-override) — same 4-arm shape and index rationale as
+ // backend/src/monitor.rs::delete_expired_jobs_batch (see there for the full rationale).
+ let (deleted_jobs, max_completed_at) = match only_workspace {
+ Some(w_id) if active_root_job_ids.is_empty() => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT id FROM v2_job_completed
+ WHERE workspace_id = $4
+ AND completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)
+ ORDER BY completed_at ASC
+ LIMIT $2
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ completed_at_floor,
+ w_id,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ Some(w_id) => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT jc.id FROM v2_job_completed jc
+ LEFT JOIN v2_job j ON j.id = jc.id
+ WHERE jc.workspace_id = $5
+ AND jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)
+ AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
+ SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
+ )
+ ORDER BY jc.completed_at ASC
+ LIMIT $2
+ FOR UPDATE OF jc SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ &active_root_job_ids,
+ completed_at_floor,
+ w_id,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ None if active_root_job_ids.is_empty() => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT id FROM v2_job_completed
+ WHERE completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND completed_at >= COALESCE($3::timestamptz, '-infinity'::timestamptz)
+ AND ($4::text[] IS NULL OR workspace_id NOT IN (
+ SELECT u FROM unnest($4::text[]) AS u WHERE u IS NOT NULL
+ ))
+ ORDER BY completed_at ASC
+ LIMIT $2
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ completed_at_floor,
+ exclude_workspaces,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
+ None => {
+ let rows = sqlx::query!(
+ "DELETE FROM v2_job_completed
+ WHERE id IN (
+ SELECT jc.id FROM v2_job_completed jc
+ LEFT JOIN v2_job j ON j.id = jc.id
+ WHERE jc.completed_at <= now() - ($1::bigint::text || ' s')::interval
+ AND jc.completed_at >= COALESCE($4::timestamptz, '-infinity'::timestamptz)
+ AND ($5::text[] IS NULL OR jc.workspace_id NOT IN (
+ SELECT u FROM unnest($5::text[]) AS u WHERE u IS NOT NULL
+ ))
+ AND COALESCE(j.root_job, j.flow_innermost_root_job, jc.id) NOT IN (
+ SELECT u FROM unnest($3::uuid[]) AS u WHERE u IS NOT NULL
+ )
+ ORDER BY jc.completed_at ASC
+ LIMIT $2
+ FOR UPDATE OF jc SKIP LOCKED
+ )
+ RETURNING id, completed_at",
+ job_retention_secs,
+ batch_size,
+ &active_root_job_ids,
+ completed_at_floor,
+ exclude_workspaces,
+ )
+ .fetch_all(&mut *tx)
+ .await?;
+ let max = rows.iter().map(|r| r.completed_at).max();
+ (rows.into_iter().map(|r| r.id).collect::>(), max)
+ }
};
let deleted_count = deleted_jobs.len();
@@ -542,15 +681,28 @@ async fn cleanup_s3_orphans(
let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed);
let now = Utc::now();
// Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS),
- // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. Job-log
- // orphans, by contrast, can only be considered expired relative to
- // JOB_RETENTION_SECS; when that is disabled we skip the job branch entirely.
+ // so we scan for service-log orphans regardless of JOB_RETENTION_SECS.
let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS);
- let job_cutoff = if job_retention_secs > 0 {
- Some(now - chrono::Duration::seconds(job_retention_secs))
- } else {
- None
- };
+
+ // Job-log orphans are only considered once past a job's effective retention window. That window
+ // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as
+ // soon as the SHORTEST applicable window elapses. Since this scan applies a single cutoff (the S3
+ // path carries only the job id, not the workspace), use the MINIMUM positive window across the
+ // instance window and every positive override so no window's orphans are missed. Crucially this
+ // also covers a `0` (keep-forever) instance window that still has positive overrides — the case
+ // where a plain global-only cutoff would skip the job branch entirely and orphan those logs
+ // forever. Keep-forever windows (0) contribute nothing: their jobs are never deleted. Overrides
+ // are folded in only once the cache is a known state; otherwise we fall back to the instance
+ // window alone and the next run picks up any override-only orphans once the cache loads.
+ let mut min_positive_window = (job_retention_secs > 0).then_some(job_retention_secs);
+ if JOB_RETENTION_SECS_OVERRIDES_LOADED.load(std::sync::atomic::Ordering::Relaxed) {
+ for w in JOB_RETENTION_SECS_OVERRIDES.load_full().values().copied() {
+ if w > 0 {
+ min_positive_window = Some(min_positive_window.map_or(w, |m| m.min(w)));
+ }
+ }
+ }
+ let job_cutoff = min_positive_window.map(|w| now - chrono::Duration::seconds(w));
let logs_prefix = ObjectPath::from("logs/");
let mut stream = store.list(Some(&logs_prefix));
diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs
index 2c28a6729a..ba359cbffb 100644
--- a/backend/windmill-api-workspaces/src/workspaces.rs
+++ b/backend/windmill-api-workspaces/src/workspaces.rs
@@ -71,9 +71,7 @@ use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, Postgres, Row, Transaction};
use windmill_common::oauth2::InstanceEvent;
-use windmill_common::secret_backend::{
- get_secret_backend, is_external_stored_value, is_vault_backend_configured,
-};
+use windmill_common::secret_backend::{get_secret_backend, is_vault_backend_configured};
use windmill_common::utils::not_found_if_none;
lazy_static::lazy_static! {
@@ -4703,15 +4701,13 @@ async fn clone_variables(
.execute(&mut **tx)
.await?;
- // With an external secret backend (Vault / Azure KV / AWS SM), the copied
- // `value` is only a `$vault:`/`$azure_kv:`/`$aws_sm:` marker: the actual
- // secret lives in the external store under a key derived from
- // (workspace_id, path). The row copy above therefore leaves the fork's
- // markers pointing at keys that don't exist — replicate each secret under
- // the fork's workspace id.
+ // With an external backend the secret lives in the store under (workspace_id,
+ // path), so the row copy above leaves the fork pointing at keys that don't
+ // exist. Replicate every secret, not just marker-valued ones: migration writes
+ // to the store without rewriting `value` to a `$...:` marker.
if is_vault_backend_configured(db).await? {
let secret_variables = sqlx::query!(
- "SELECT path, value FROM variable
+ "SELECT path FROM variable
WHERE workspace_id = $1 AND is_secret = true AND value != ''",
target_workspace_id,
)
@@ -4719,10 +4715,7 @@ async fn clone_variables(
.await?;
let backend = get_secret_backend(db).await?;
- for variable in secret_variables
- .into_iter()
- .filter(|v| is_external_stored_value(&v.value))
- {
+ for variable in secret_variables {
match backend
.get_secret(source_workspace_id, &variable.path)
.await
@@ -5810,8 +5803,14 @@ async fn create_workspace_fork(
.await?;
// Clone all data from the parent workspace using Rust implementation
- if let Err(e) =
- clone_workspace_data(&mut tx, &db, &parent_workspace_id, &forked_id, &authed.email).await
+ if let Err(e) = clone_workspace_data(
+ &mut tx,
+ &db,
+ &parent_workspace_id,
+ &forked_id,
+ &authed.email,
+ )
+ .await
{
// A genuine `\u0000` in a source `json` value (`app_version.value` /
// `flow_version.schema`) aborts the clone when it is re-encoded to jsonb:
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index 6a214e8d87..181b5e8733 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
- version: 1.753.0
+ version: 1.757.0
title: Windmill API
contact:
@@ -11924,7 +11924,7 @@ paths:
/w/{workspace}/apps/sign_s3_objects:
post:
- summary: sign s3 objects, to be used by anonymous users in public apps
+ summary: sign s3 objects (caller must have S3 read permission on each key); the signed URLs can then be used by anonymous users in public apps
operationId: signS3Objects
tags:
- app
@@ -12121,6 +12121,247 @@ paths:
schema:
type: string
+ /w/{workspace}/apps_u/load_file_metadata/{path}:
+ get:
+ summary: Load metadata of an s3 file on-behalf of the app author (deployed app)
+ operationId: appLoadFileMetadata
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: storage
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: FileMetadata
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/WindmillFileMetadata"
+
+ /w/{workspace}/apps_u/load_file_preview/{path}:
+ get:
+ summary: Load a preview of an s3 file on-behalf of the app author (deployed app)
+ operationId: appLoadFilePreview
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: file_size_in_bytes
+ in: query
+ schema:
+ type: integer
+ - name: file_mime_type
+ in: query
+ schema:
+ type: string
+ - name: csv_separator
+ in: query
+ schema:
+ type: string
+ - name: csv_has_header
+ in: query
+ schema:
+ type: boolean
+ - name: read_bytes_from
+ in: query
+ required: true
+ schema:
+ type: integer
+ - name: read_bytes_length
+ in: query
+ required: true
+ schema:
+ type: integer
+ - name: storage
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: FilePreview
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/WindmillFilePreview"
+
+ /w/{workspace}/apps_u/load_parquet_preview/{path}:
+ get:
+ summary: Load a preview of a parquet file on-behalf of the app author (deployed app)
+ operationId: appLoadParquetPreview
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: offset
+ in: query
+ schema:
+ type: number
+ - name: limit
+ in: query
+ schema:
+ type: number
+ - name: sort_col
+ in: query
+ schema:
+ type: string
+ - name: sort_desc
+ in: query
+ schema:
+ type: boolean
+ - name: search_col
+ in: query
+ schema:
+ type: string
+ - name: search_term
+ in: query
+ schema:
+ type: string
+ - name: storage
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Parquet Preview
+ content:
+ application/json: {}
+
+ /w/{workspace}/apps_u/load_csv_preview/{path}:
+ get:
+ summary: Load a preview of a csv file on-behalf of the app author (deployed app)
+ operationId: appLoadCsvPreview
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: offset
+ in: query
+ schema:
+ type: number
+ - name: limit
+ in: query
+ schema:
+ type: number
+ - name: sort_col
+ in: query
+ schema:
+ type: string
+ - name: sort_desc
+ in: query
+ schema:
+ type: boolean
+ - name: search_col
+ in: query
+ schema:
+ type: string
+ - name: search_term
+ in: query
+ schema:
+ type: string
+ - name: storage
+ in: query
+ schema:
+ type: string
+ - name: csv_separator
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Csv Preview
+ content:
+ application/json: {}
+
+ /w/{workspace}/apps_u/load_table_count/{path}:
+ get:
+ summary: Load the table row count on-behalf of the app author (deployed app)
+ operationId: appLoadTableCount
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: search_col
+ in: query
+ schema:
+ type: string
+ - name: search_term
+ in: query
+ schema:
+ type: string
+ - name: storage
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: Table count
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ count:
+ type: number
+
+ /w/{workspace}/apps_u/download_s3_parquet_file_as_csv/{path}:
+ get:
+ summary: Download a parquet s3 file as csv on-behalf of the app author (deployed app)
+ operationId: appDownloadS3ParquetFileAsCsv
+ tags:
+ - app
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ - $ref: "#/components/parameters/Path"
+ - name: file_key
+ in: query
+ required: true
+ schema:
+ type: string
+ - name: storage
+ in: query
+ schema:
+ type: string
+ responses:
+ "200":
+ description: The downloaded file
+ content:
+ text/csv:
+ schema:
+ type: string
+
/w/{workspace}/jobs/run/f/{path}:
post:
summary: run flow by path
@@ -26059,6 +26300,7 @@ components:
- github
- asset
- freshness
+ - app
TriggerMode:
description: job trigger mode
diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs
index a2ad8f3c3e..596d28ed64 100644
--- a/backend/windmill-api/src/apps.rs
+++ b/backend/windmill-api/src/apps.rs
@@ -26,6 +26,7 @@ use crate::{
job_helpers_oss::{
download_s3_file_internal, get_random_file_name, get_s3_resource,
get_workspace_s3_resource_and_check_paths, upload_file_from_req, DownloadFileQuery,
+ LoadCountQuery, LoadFileMetadataQuery, LoadFilePreviewQuery, LoadPreviewQuery,
},
users::fetch_api_authed_from_permissioned_as,
};
@@ -61,8 +62,9 @@ use windmill_common::{
error::{to_anyhow, Error, JsonResult, Result},
jobs::{
get_payload_tag_from_prefixed_path, resolve_delete_after_secs, schedule_job_deletion,
- JobPayload, RawCode,
+ JobPayload, JobTriggerKind, RawCode,
},
+ triggers::TriggerMetadata,
user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay},
users::username_to_permissioned_as,
utils::{
@@ -140,6 +142,18 @@ pub fn unauthed_service() -> Router {
.route("/upload_s3_file/{*path}", post(upload_s3_file_from_app))
.route("/delete_s3_file", delete(delete_s3_file_from_app))
.route("/download_s3_file/{*path}", get(download_s3_file_from_app))
+ .route(
+ "/download_s3_parquet_file_as_csv/{*path}",
+ get(app_download_s3_parquet_file_as_csv),
+ )
+ .route("/load_file_metadata/{*path}", get(app_load_file_metadata))
+ .route("/load_file_preview/{*path}", get(app_load_file_preview))
+ .route("/load_table_count/{*path}", get(app_load_table_count))
+ .route(
+ "/load_parquet_preview/{*path}",
+ get(app_load_parquet_preview),
+ )
+ .route("/load_csv_preview/{*path}", get(app_load_csv_preview))
.route("/public_app/{secret}", get(get_public_app_by_secret))
.route("/embed_token/{secret}", get(get_app_embed_token))
.route("/public_resource/{*path}", get(get_public_resource))
@@ -3121,23 +3135,26 @@ async fn execute_component(
}
.filter(|t| !t.is_empty())
};
- let (job_payload, tag, on_behalf_of) = match (payload.path, payload.raw_code, payload.id) {
- // flow or script:
- (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
- // inline script: "preview" mode, or run mode without an entry in the
- // `app_script` table (legacy `rawscript/`-keyed triggerables).
- (None, Some(raw_code), None) => {
- let tag = resolved_inline_tag(raw_code.tag.clone());
- (JobPayload::Code(raw_code), tag, None)
- }
- // inline script: run mode (deployed app) with an entry in `app_script`.
- (None, Some(RawCode { language, path, cache_ttl, tag, .. }), Some(id)) => (
- JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
- resolved_inline_tag(tag),
- None,
- ),
- _ => unreachable!(),
- };
+ let (job_payload, tag, _runnable_on_behalf_of) =
+ match (payload.path, payload.raw_code, payload.id) {
+ // flow or script:
+ (Some(path), None, None) => {
+ get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?
+ }
+ // inline script: "preview" mode, or run mode without an entry in the
+ // `app_script` table (legacy `rawscript/`-keyed triggerables).
+ (None, Some(raw_code), None) => {
+ let tag = resolved_inline_tag(raw_code.tag.clone());
+ (JobPayload::Code(raw_code), tag, None)
+ }
+ // inline script: run mode (deployed app) with an entry in `app_script`.
+ (None, Some(RawCode { language, path, cache_ttl, tag, .. }), Some(id)) => (
+ JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
+ resolved_inline_tag(tag),
+ None,
+ ),
+ _ => unreachable!(),
+ };
// Preview honors the client-supplied inline tag (`resolved_inline_tag`), so
// — like `/jobs/run/preview` — confine it to worker tags the caller may use
// (a `if_jobs:filter_tags`-restricted token must not escape its filter).
@@ -3154,18 +3171,22 @@ async fn execute_component(
// and would add unnecessary breakage risk to the legitimate editor flow.
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
- let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() {
- (
- on_behalf_of.email.as_str(),
- on_behalf_of.permissioned_as.clone(),
- )
- } else {
- (email.as_str(), permissioned_as)
- };
+ // An app component runs on-behalf of the APP identity (resolved above), never
+ // the referenced runnable's own `on_behalf_of` — else a Viewer-mode app could
+ // execute as that identity and a preview would run as it, not the caller.
+ // (Direct `/jobs/run` still honors a runnable's `on_behalf_of`.)
+ let (email, permissioned_as) = (email.as_str(), permissioned_as);
let end_user_email =
get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await;
+ // Stamp app-origination (trigger_kind='app' + trigger=), the signal
+ // the deployed-app S3 provenance gate trusts (unforgeable via `/jobs/run`).
+ // Deployed runs only: a preview runs as the caller and is read back as the caller
+ // (viewer-scoped), so it must never be app-provenanced (else it could forge one).
+ let app_trigger =
+ (!is_preview).then(|| TriggerMetadata::new(Some(path.to_string()), JobTriggerKind::App));
+
let (uuid, mut tx) = push(
&db,
tx,
@@ -3196,7 +3217,7 @@ async fn execute_component(
None,
false,
end_user_email,
- None,
+ app_trigger,
None,
)
.await?;
@@ -3266,6 +3287,7 @@ struct S3TokenRequestBody {
}
#[cfg(feature = "parquet")]
async fn sign_s3_objects(
+ authed: ApiAuthed,
Extension(db): Extension,
Path(w_id): Path,
Json(body): Json,
@@ -3273,6 +3295,22 @@ async fn sign_s3_objects(
let workspace_key = get_workspace_key(&w_id, &db).await?;
let futures = body.s3_objects.into_iter().map(|s3_object| async {
+ // The signature this mints is a transferable bearer capability: `validate_s3_signature`
+ // only checks the HMAC and expiry, so anyone who obtains the string can read this key.
+ // Authorize the CALLER's own read permission before signing — otherwise any workspace
+ // member (operators included) could mint a signature for any key and bypass the advanced
+ // S3 permission rules. This is the fix; do NOT move the check to validation time.
+ let db_with_opt_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), None);
+ get_workspace_s3_resource_and_check_paths(
+ &db_with_opt_authed,
+ Some(&authed),
+ &w_id,
+ s3_object.storage.clone(),
+ &[(&s3_object.s3, S3Permission::READ)],
+ None,
+ )
+ .await?;
+
let exp = (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp();
let mut message = format!("file_key={}&exp={}", s3_object.s3.clone(), exp);
if let Some(ref storage) = s3_object.storage {
@@ -3808,8 +3846,9 @@ async fn check_if_allowed_to_access_s3_file_from_app(
path: &str,
policy: &Policy,
) -> Result<()> {
- // if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours
- // otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy)
+ let is_app_embed = opt_authed.as_ref().is_some_and(|authed| {
+ windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref())
+ });
if file_query.sig.is_some() {
#[cfg(feature = "private")]
@@ -3829,19 +3868,19 @@ async fn check_if_allowed_to_access_s3_file_from_app(
return Err(Error::InternalErr(
"Internal error: signature validation is not supported in open source mode".to_string(),
));
- } else if opt_authed.as_ref().is_some_and(|authed| {
- !windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref())
- }) {
- // A normal logged-in caller (editor / full session) may fetch any file they
- // can reach. An app embed token also carries an identity but represents
- // untrusted app JS, so it falls through to the allowlist below instead of
- // this bypass — otherwise the app could read arbitrary S3 keys the
- // viewer/on-behalf identity can see, beyond its own declared keys/outputs.
+ } else if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed {
+ // Viewer mode: the on-behalf identity IS the viewer, so the downstream
+ // get_workspace_s3_resource_and_check_paths already bounds the read by
+ // their own perms — no provenance gate (it would over-restrict). Embed
+ // tokens are excluded (untrusted app JS stays confined below).
Ok(())
} else {
- // Anonymous viewer, or an app embed token: confine to the app's declared S3
- // keys, or files produced by THIS app's own component runs. The producing
- // identity is the embed viewer for a token, else `anonymous`.
+ // Author-mode/embed: confine reads to the app's declared keys or files THIS
+ // app produced, else a viewer could launder the author's S3 perms via an
+ // arbitrary file_key (confused deputy). Provenance is the un-forgeable
+ // app-origination marker (`trigger_kind='app'` + `trigger=`);
+ // `created_by=` is ANDed only as a per-viewer isolation filter (it
+ // can narrow — one viewer can't read another's result — never forge).
let creator = opt_authed
.as_ref()
.map(|authed| authed.username.clone())
@@ -3854,11 +3893,11 @@ async fn check_if_allowed_to_access_s3_file_from_app(
r#"SELECT EXISTS (
SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.workspace_id = $2
- AND (j.kind = 'appscript' OR j.kind = 'preview')
- AND j.created_by = $4
AND c.started_at > now() - interval '3 hours'
- AND j.runnable_path LIKE $3 || '/%'
AND c.result @> ('{"s3":"' || $1 || '"}')::jsonb
+ AND j.trigger_kind = 'app'
+ AND j.trigger = $3
+ AND j.created_by = $4
)"#,
file_query.s3,
w_id,
@@ -3966,6 +4005,258 @@ async fn download_s3_file_from_app(
.await
}
+#[cfg(feature = "parquet")]
+fn app_s3_file_query(s3: String, storage: Option) -> AppS3FileQuery {
+ AppS3FileQuery {
+ s3,
+ storage,
+ sig: None,
+ #[cfg(feature = "private")]
+ exp: None,
+ }
+}
+
+/// Shared entry for every app-scoped (`apps_u/*`) S3 display op: scope-confine an
+/// app embed token, resolve the on-behalf identity per `execution_mode`, then run
+/// the provenance gate (`check_if_allowed_to_access_s3_file_from_app`) once before
+/// dispatching to the S3 helpers.
+#[cfg(feature = "parquet")]
+async fn app_s3_on_behalf_and_provenance(
+ db: &DB,
+ path: &str,
+ w_id: &str,
+ opt_authed: &Option,
+ file_query: &AppS3FileQuery,
+) -> Result {
+ if let Some(authed) = opt_authed.as_ref() {
+ check_scopes(authed, || format!("apps:read:{}", path))?;
+ }
+ let (on_behalf_authed, policy) =
+ get_on_behalf_authed_from_app(db, path, w_id, opt_authed, None).await?;
+ check_if_allowed_to_access_s3_file_from_app(db, opt_authed, file_query, w_id, path, &policy)
+ .await?;
+ Ok(crate::db::OptJobAuthed { authed: on_behalf_authed, job_id: None })
+}
+
+// The app-scoped display ops carry the app path in the URL and everything else
+// (file_key + op args) in the query, so they avoid a second `{*path}` wildcard.
+// `LoadCountQuery` / `LoadPreviewQuery` don't include the file key (it's a path
+// param on the raw `job_helpers/*` route), so restate their fields here with the
+// file key added. Do NOT `#[serde(flatten)]` the inner struct: axum's `Query`
+// uses `serde_urlencoded`, which cannot deserialize a flattened field's typed
+// (numeric/bool) values and 400s on `limit`/`offset` — the fields must be
+// declared directly on the outer struct.
+#[cfg(feature = "parquet")]
+#[derive(Deserialize)]
+struct AppLoadCountQuery {
+ file_key: String,
+ search_col: Option,
+ search_term: Option,
+ storage: Option,
+}
+
+#[cfg(feature = "parquet")]
+impl AppLoadCountQuery {
+ fn into_inner(self) -> (String, LoadCountQuery) {
+ (
+ self.file_key,
+ LoadCountQuery {
+ search_col: self.search_col,
+ search_term: self.search_term,
+ storage: self.storage,
+ },
+ )
+ }
+}
+
+#[cfg(feature = "parquet")]
+#[derive(Deserialize)]
+struct AppLoadPreviewQuery {
+ file_key: String,
+ limit: Option,
+ offset: Option,
+ sort_col: Option,
+ sort_desc: Option,
+ search_col: Option,
+ search_term: Option,
+ storage: Option,
+ csv_separator: Option,
+}
+
+#[cfg(feature = "parquet")]
+impl AppLoadPreviewQuery {
+ fn into_inner(self) -> (String, LoadPreviewQuery) {
+ (
+ self.file_key,
+ LoadPreviewQuery {
+ limit: self.limit,
+ offset: self.offset,
+ sort_col: self.sort_col,
+ sort_desc: self.sort_desc,
+ search_col: self.search_col,
+ search_term: self.search_term,
+ storage: self.storage,
+ csv_separator: self.csv_separator,
+ },
+ )
+ }
+}
+
+#[cfg(feature = "parquet")]
+async fn app_download_s3_parquet_file_as_csv(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ crate::job_helpers_oss::download_s3_parquet_file_as_csv_internal(
+ job_authed,
+ &db,
+ None,
+ &w_id,
+ DownloadFileQuery {
+ file_key: query.file_key,
+ s3_resource_path: None,
+ storage: query.storage,
+ },
+ )
+ .await
+}
+
+#[cfg(feature = "parquet")]
+async fn app_load_file_metadata(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ let resp =
+ crate::job_helpers_oss::load_file_metadata_internal(job_authed, &db, &w_id, query).await?;
+ Ok(Json(resp).into_response())
+}
+
+#[cfg(feature = "parquet")]
+async fn app_load_file_preview(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ let resp =
+ crate::job_helpers_oss::load_file_preview_internal(job_authed, &db, &w_id, query).await?;
+ Ok(Json(resp).into_response())
+}
+
+#[cfg(feature = "parquet")]
+async fn app_load_table_count(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let (file_key, inner) = query.into_inner();
+ let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ let resp =
+ crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner)
+ .await?;
+ Ok(Json(resp).into_response())
+}
+
+#[cfg(feature = "parquet")]
+async fn app_load_parquet_preview(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let (file_key, inner) = query.into_inner();
+ let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ let resp = crate::job_helpers_oss::load_preview_internal(
+ job_authed, &db, &w_id, file_key, inner, true,
+ )
+ .await?;
+ Ok(Json(resp).into_response())
+}
+
+#[cfg(feature = "parquet")]
+async fn app_load_csv_preview(
+ OptAuthed(opt_authed): OptAuthed,
+ Extension(db): Extension,
+ Path((w_id, path)): Path<(String, StripPath)>,
+ Query(query): Query,
+) -> Result {
+ let path = path.to_path();
+ let (file_key, inner) = query.into_inner();
+ let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone());
+ let job_authed =
+ app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?;
+ let resp = crate::job_helpers_oss::load_preview_internal(
+ job_authed, &db, &w_id, file_key, inner, false,
+ )
+ .await?;
+ Ok(Json(resp).into_response())
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_download_s3_parquet_file_as_csv() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_load_file_metadata() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_load_file_preview() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_load_table_count() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_load_parquet_preview() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
+#[cfg(not(feature = "parquet"))]
+async fn app_load_csv_preview() -> Result<()> {
+ Err(Error::BadRequest(
+ "This endpoint requires the parquet feature to be enabled".to_string(),
+ ))
+}
+
fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
let permissioned_as = policy
.on_behalf_of
diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs
index 8b9489263b..c564548b42 100644
--- a/backend/windmill-api/src/db.rs
+++ b/backend/windmill-api/src/db.rs
@@ -90,6 +90,9 @@ lazy_static::lazy_static! {
(20260614075900, include_str!(
"../../migrations/20260614075900_dedup_folder_labels.up.sql"
).replace("SET search_path = public", "SET search_path FROM CURRENT").to_string()),
+ (20260710073406, include_str!(
+ "../../migrations/20260710073406_index_v2_job_parent_job.up.sql"
+ ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")),
].into_iter().collect();
}
diff --git a/backend/windmill-api/src/db_health.rs b/backend/windmill-api/src/db_health.rs
index c487d29da9..0b2e86567b 100644
--- a/backend/windmill-api/src/db_health.rs
+++ b/backend/windmill-api/src/db_health.rs
@@ -15,6 +15,7 @@ use axum::{
use serde::{Deserialize, Serialize};
use windmill_common::error::JsonResult;
+use windmill_common::{JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED};
use crate::db::{ApiAuthed, DB};
use crate::utils::require_super_admin;
@@ -291,10 +292,50 @@ async fn fetch_database_size(db: &DB) -> windmill_common::error::Result windmill_common::error::Result {
- let job_row =
- sqlx::query!("SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed")
- .fetch_one(db)
- .await?;
+ // Per-workspace retention overrides (EE) make "oldest completed job vs the instance retention"
+ // wrong as a single global signal: each override workspace has its own effective window, so its
+ // intentionally-retained jobs must be judged against that window — not the instance one. We
+ // therefore compute the ratio per scope and report the worst:
+ // - global scope: oldest job across all non-override workspaces vs the instance retention;
+ // - each override workspace with a *positive* window: its own oldest job vs its own window;
+ // - keep-forever (0) overrides: excluded entirely — their jobs are retained forever by design,
+ // so there is no window to fall behind on.
+ // The majority (no-override) path keeps the original index-driven `MIN(completed_at)` with no
+ // performance change; the override paths use the completed_at / (workspace_id, completed_at)
+ // indexes and only run when overrides exist.
+ let overrides = JOB_RETENTION_SECS_OVERRIDES.load_full();
+ let overrides_active = !overrides.is_empty()
+ && JOB_RETENTION_SECS_OVERRIDES_LOADED.load(std::sync::atomic::Ordering::Relaxed);
+
+ // `true_oldest` is the real table minimum across every workspace — reported verbatim in the
+ // public `oldest_completed_at` field so the UI's "Oldest job" label stays honest. `global_oldest`
+ // excludes override workspaces and drives only the global health ratio (override workspaces are
+ // judged against their own window below).
+ type OptTs = Option>;
+ let (true_oldest, global_oldest, total): (OptTs, OptTs, i64) = if !overrides_active {
+ let r = sqlx::query!(
+ "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed"
+ )
+ .fetch_one(db)
+ .await?;
+ (r.oldest, r.oldest, r.total.unwrap_or(0))
+ } else {
+ // `MIN(...) WHERE workspace_id <> ALL(...)` is still driven by the completed_at index
+ // (ascending scan, early-stop at the first non-override row); the plain `MIN(...)` is an
+ // index-only scan. Both are cheap.
+ let override_ids: Vec = overrides.keys().cloned().collect();
+ let r = sqlx::query!(
+ "SELECT
+ (SELECT MIN(completed_at) FROM v2_job_completed) as true_oldest,
+ (SELECT MIN(completed_at) FROM v2_job_completed
+ WHERE workspace_id <> ALL($1::text[])) as global_oldest,
+ (SELECT COUNT(*) FROM v2_job_completed) as total",
+ &override_ids,
+ )
+ .fetch_one(db)
+ .await?;
+ (r.true_oldest, r.global_oldest, r.total.unwrap_or(0))
+ };
let retention_row =
sqlx::query!("SELECT value FROM global_settings WHERE name = 'retention_period_secs'")
@@ -304,48 +345,103 @@ async fn fetch_job_retention(db: &DB) -> windmill_common::error::Result =
retention_row.map(|r| r.value).and_then(|v| v.as_i64());
- let oldest = job_row.oldest;
- let total = job_row.total.unwrap_or(0);
+ // Oldest job per positive-window override workspace (one grouped seek on the
+ // `(workspace_id, completed_at)` index; only workspaces that actually have rows come back).
+ let positive_override_ids: Vec = if overrides_active {
+ overrides
+ .iter()
+ .filter(|(_, &secs)| secs > 0)
+ .map(|(ws, _)| ws.clone())
+ .collect()
+ } else {
+ Vec::new()
+ };
+ let mut per_workspace_oldest: Vec<(String, chrono::DateTime)> = Vec::new();
+ if !positive_override_ids.is_empty() {
+ let rows = sqlx::query!(
+ "SELECT workspace_id as \"workspace_id!\", MIN(completed_at) as oldest
+ FROM v2_job_completed
+ WHERE workspace_id = ANY($1::text[])
+ GROUP BY workspace_id",
+ &positive_override_ids,
+ )
+ .fetch_all(db)
+ .await?;
+ for r in rows {
+ if let Some(oldest) = r.oldest {
+ per_workspace_oldest.push((r.workspace_id, oldest));
+ }
+ }
+ }
- let (status, message) = if let (Some(oldest_ts), Some(retention_secs)) =
- (oldest, retention_period_secs)
- {
- let age_secs: i64 = (chrono::Utc::now() - oldest_ts).num_seconds();
- let ratio = if retention_secs > 0 {
- age_secs as f64 / retention_secs as f64
- } else {
- 0.0
- };
+ // Evaluate each scope independently and report the WORST. Each scope contributes a candidate
+ // (severity, level, message); the max-severity candidate wins. This keeps the global scope's
+ // "no retention configured" warning visible even when a healthy override would otherwise mask it.
+ let now = chrono::Utc::now();
+ let ratio_status = |scope: String, ratio: f64| -> (u8, HealthLevel, String) {
if ratio <= 2.0 {
(
+ 0,
HealthLevel::Green,
- format!(
- "Oldest job is {:.1}x the retention period. Cleanup is keeping up.",
- ratio
- ),
+ format!("{scope} is {ratio:.1}x the retention period. Cleanup is keeping up."),
)
} else if ratio <= 5.0 {
(
+ 1,
HealthLevel::Yellow,
format!(
- "Oldest job is {:.1}x the retention period. Cleanup may be falling behind.",
- ratio
+ "{scope} is {ratio:.1}x the retention period. Cleanup may be falling behind."
),
)
} else {
- (HealthLevel::Red, format!("Oldest job is {:.1}x the retention period. Consider reducing retention or investigating cleanup.", ratio))
+ (2, HealthLevel::Red, format!("{scope} is {ratio:.1}x the retention period. Consider reducing retention or investigating cleanup."))
}
- } else if oldest.is_some() && retention_period_secs.is_none() {
- (
+ };
+
+ let mut candidates: Vec<(u8, HealthLevel, String)> = Vec::new();
+ // Global scope: judged against the instance retention, or flagged when non-override jobs exist
+ // (`global_oldest` is `Some`) but no positive instance retention is configured. A `0` instance
+ // retention means keep-forever globally, so it contributes no candidate.
+ match (global_oldest, retention_period_secs) {
+ (Some(oldest_ts), Some(retention_secs)) if retention_secs > 0 => {
+ let ratio = (now - oldest_ts).num_seconds() as f64 / retention_secs as f64;
+ candidates.push(ratio_status("Oldest job".to_string(), ratio));
+ }
+ (Some(_), None) => candidates.push((
+ 1,
HealthLevel::Yellow,
"No retention_period_secs configured. Old jobs will accumulate.".to_string(),
+ )),
+ _ => {}
+ }
+ // Each positive-window override, judged against its own window.
+ for (ws, oldest_ts) in &per_workspace_oldest {
+ if let Some(&window) = overrides.get(ws) {
+ if window > 0 {
+ let ratio = (now - *oldest_ts).num_seconds() as f64 / window as f64;
+ candidates.push(ratio_status(format!("Workspace {ws} oldest job"), ratio));
+ }
+ }
+ }
+
+ let (status, message) = if let Some((_, level, message)) = candidates
+ .into_iter()
+ .max_by_key(|(severity, _, _)| *severity)
+ {
+ (level, message)
+ } else if total > 0 {
+ // Jobs exist but none produced a candidate: every completed job lives in a keep-forever
+ // scope (instance or override), so it is retained by design rather than overdue.
+ (
+ HealthLevel::Green,
+ "Completed jobs are within their configured retention windows.".to_string(),
)
} else {
(HealthLevel::Green, "No completed jobs found.".to_string())
};
Ok(JobRetentionInfo {
- oldest_completed_at: oldest,
+ oldest_completed_at: true_oldest,
total_completed_jobs: total,
retention_period_secs,
status,
diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs
index 72791de78f..ffc94d5b46 100644
--- a/backend/windmill-api/src/drafts.rs
+++ b/backend/windmill-api/src/drafts.rs
@@ -120,7 +120,11 @@ async fn list_drafts(
.await
{
Ok(()) => true,
- Err(Error::NotAuthorized(_)) => false,
+ // A stored draft can sit at an unwritable path — unauthorized,
+ // or malformed (`BadRequest`; the `draft` table has no path
+ // constraint). Either way it's not writable, and one bad row
+ // must not 400 the whole listing.
+ Err(Error::NotAuthorized(_)) | Err(Error::BadRequest(_)) => false,
Err(e) => return Err(e),
};
out.push(row);
@@ -745,8 +749,17 @@ async fn require_can_write_path(
return Ok(());
}
}
+ // A path without a recognized namespace prefix (u/, f/, g/) can never be
+ // writable — no namespace rule and no deployed row can apply — so report it
+ // as malformed rather than as a plain permission denial.
+ if !(path.starts_with("u/") || path.starts_with("f/") || path.starts_with("g/")) {
+ return Err(Error::BadRequest(format!(
+ "Invalid path '{path}': a valid path starts with 'u//', 'f//' or 'g//'"
+ )));
+ }
Err(Error::NotAuthorized(format!(
- "you don't have write permission on {path}"
+ "You don't have write permission on '{path}'. It must be in your own 'u/{}/' namespace, or in a folder ('f//') or group ('g//') you can write to.",
+ authed.username
)))
}
diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs
index c6d8e397f2..8a50f2e481 100644
--- a/backend/windmill-api/src/job_helpers_oss.rs
+++ b/backend/windmill-api/src/job_helpers_oss.rs
@@ -217,6 +217,130 @@ pub struct DeleteS3FileQuery {
pub storage: Option,
}
+// Stubs for the app-scoped S3 display ops (mirrors the EE `*_internal` helpers +
+// their query/response structs). Only compiled for a CE build with `parquet` but
+// without `private`; the real implementations live in `job_helpers_ee.rs`.
+#[cfg(all(feature = "parquet", not(feature = "private")))]
+mod app_s3_display_stubs {
+ use super::*;
+ use serde::Serialize;
+ use serde_json::value::RawValue;
+
+ #[derive(Deserialize)]
+ #[allow(dead_code)]
+ pub struct LoadFileMetadataQuery {
+ pub file_key: String,
+ pub storage: Option,
+ }
+
+ #[derive(Serialize)]
+ pub struct LoadFileMetadataResponse {}
+
+ // Mirror the EE query's required/optional fields so the CE build enforces the
+ // same query contract (e.g. the mandatory byte range) at the extraction layer.
+ #[derive(Deserialize)]
+ #[allow(dead_code)]
+ pub struct LoadFilePreviewQuery {
+ pub storage: Option,
+ pub file_key: String,
+ pub file_size_in_bytes: Option,
+ pub file_mime_type: Option,
+ pub csv_separator: Option,
+ pub csv_has_header: Option,
+ pub read_bytes_from: u64,
+ pub read_bytes_length: u64,
+ }
+
+ #[derive(Serialize)]
+ pub struct LoadFilePreviewResponse {}
+
+ #[derive(Deserialize)]
+ #[allow(dead_code)]
+ pub struct LoadCountQuery {
+ pub search_col: Option,
+ pub search_term: Option,
+ pub storage: Option,
+ }
+
+ #[derive(Serialize)]
+ pub struct TableCount {}
+
+ #[derive(Deserialize)]
+ #[allow(dead_code)]
+ pub struct LoadPreviewQuery {
+ pub limit: Option,
+ pub offset: Option,
+ pub sort_col: Option,
+ pub sort_desc: Option,
+ pub search_col: Option,
+ pub search_term: Option,
+ pub storage: Option,
+ pub csv_separator: Option,
+ }
+
+ pub async fn load_file_metadata_internal(
+ _authed: OptJobAuthed,
+ _db: &DB,
+ _w_id: &str,
+ _query: LoadFileMetadataQuery,
+ ) -> error::Result {
+ Err(error::Error::internal_err(
+ "Not implemented in Windmill's Open Source repository".to_string(),
+ ))
+ }
+
+ pub async fn load_file_preview_internal(
+ _authed: OptJobAuthed,
+ _db: &DB,
+ _w_id: &str,
+ _query: LoadFilePreviewQuery,
+ ) -> error::Result {
+ Err(error::Error::internal_err(
+ "Not implemented in Windmill's Open Source repository".to_string(),
+ ))
+ }
+
+ pub async fn load_table_count_internal(
+ _authed: OptJobAuthed,
+ _db: &DB,
+ _w_id: &str,
+ _file_key: String,
+ _query: LoadCountQuery,
+ ) -> error::Result {
+ Err(error::Error::internal_err(
+ "Not implemented in Windmill's Open Source repository".to_string(),
+ ))
+ }
+
+ pub async fn load_preview_internal(
+ _authed: OptJobAuthed,
+ _db: &DB,
+ _w_id: &str,
+ _file_key: String,
+ _query: LoadPreviewQuery,
+ _is_parquet: bool,
+ ) -> error::Result> {
+ Err(error::Error::internal_err(
+ "Not implemented in Windmill's Open Source repository".to_string(),
+ ))
+ }
+
+ pub async fn download_s3_parquet_file_as_csv_internal(
+ _authed: OptJobAuthed,
+ _db: &DB,
+ _user_db: Option,
+ _w_id: &str,
+ _query: DownloadFileQuery,
+ ) -> error::Result {
+ Err(error::Error::internal_err(
+ "Not implemented in Windmill's Open Source repository".to_string(),
+ ))
+ }
+}
+
+#[cfg(all(feature = "parquet", not(feature = "private")))]
+pub use app_s3_display_stubs::*;
+
#[cfg(not(feature = "private"))]
pub async fn get_workspace_s3_resource_and_check_paths<'c>(
_db_with_opt_authed: &DbWithOptAuthed<'c, ApiAuthed>,
diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs
index 73593c595a..1c6bcafdc6 100644
--- a/backend/windmill-api/src/lib.rs
+++ b/backend/windmill-api/src/lib.rs
@@ -486,8 +486,13 @@ pub async fn run_server(
add_www_authenticate_header, add_www_authenticate_header_gateway,
extract_workspace_from_token,
};
- let (mcp_router, mcp_cancellation_token) =
- setup_mcp_server(db.clone(), user_db, _base_internal_url.clone()).await?;
+ let (mcp_router, mcp_cancellation_token) = setup_mcp_server(
+ db.clone(),
+ user_db,
+ _base_internal_url.clone(),
+ auth_cache.clone(),
+ )
+ .await?;
// Workspace-scoped MCP router
let workspaced_mcp_router = mcp_router
.clone()
diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs
index 52fec66096..db1644379d 100644
--- a/backend/windmill-api/src/mcp/core.rs
+++ b/backend/windmill-api/src/mcp/core.rs
@@ -10,10 +10,11 @@ use windmill_common::{db::UserDB, utils::StripPath, DB};
use windmill_mcp::common::schema::enrich_resource_schemas;
use windmill_mcp::common::transform::apply_key_transformation;
use windmill_mcp::common::types::{
- FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
+ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo,
};
use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend};
+use crate::auth::AuthCache;
use crate::db::ApiAuthed;
use crate::jobs::{
run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery,
@@ -31,7 +32,8 @@ use std::time::Duration;
use tokio_util::sync::CancellationToken;
use windmill_mcp::server::{
- LocalSessionManager, Runner, StreamableHttpServerConfig, StreamableHttpService,
+ LocalSessionManager, McpToken, MultiWorkspaceMcp, Runner, StreamableHttpServerConfig,
+ StreamableHttpService,
};
use windmill_mcp::WorkspaceId;
@@ -53,11 +55,17 @@ pub struct WindmillBackend {
pub db: DB,
pub user_db: UserDB,
pub base_internal_url: String,
+ pub auth_cache: Arc,
}
impl WindmillBackend {
- pub fn new(db: DB, user_db: UserDB, base_internal_url: String) -> Self {
- Self { db, user_db, base_internal_url }
+ pub fn new(
+ db: DB,
+ user_db: UserDB,
+ base_internal_url: String,
+ auth_cache: Arc,
+ ) -> Self {
+ Self { db, user_db, base_internal_url, auth_cache }
}
}
@@ -305,6 +313,8 @@ impl McpBackend for WindmillBackend {
args_map,
&endpoint_tool.body_schema,
&endpoint_tool.body_field_renames,
+ &endpoint_tool.path_params_schema,
+ &endpoint_tool.query_params_schema,
);
// Create and execute request
@@ -338,6 +348,57 @@ impl McpBackend for WindmillBackend {
}
}
+ async fn list_accessible_workspaces(
+ &self,
+ auth: &ApiAuthed,
+ ) -> BackendResult> {
+ // A superadmin can act in every workspace and often has no explicit `usr`
+ // membership row (matching resolve_workspace_auth, which authorizes any
+ // workspace for a superadmin), so list them all. Everyone else is limited
+ // to the workspaces they are a member of.
+ let workspaces = if auth.is_admin {
+ sqlx::query_as!(
+ WorkspaceInfo,
+ "SELECT id, name FROM workspace WHERE deleted = false ORDER BY name",
+ )
+ .fetch_all(&self.db)
+ .await
+ } else {
+ sqlx::query_as!(
+ WorkspaceInfo,
+ "SELECT workspace.id, workspace.name
+ FROM workspace
+ JOIN usr ON usr.workspace_id = workspace.id
+ WHERE usr.email = $1 AND usr.disabled = false AND workspace.deleted = false
+ ORDER BY workspace.name",
+ auth.email,
+ )
+ .fetch_all(&self.db)
+ .await
+ };
+
+ workspaces.map_err(|e| ErrorData::internal_error(e.to_string(), None))
+ }
+
+ async fn resolve_workspace_auth(
+ &self,
+ token: &str,
+ workspace_id: &str,
+ ) -> BackendResult {
+ self.auth_cache
+ .get_authed(Some(workspace_id.to_string()), token)
+ .await
+ .ok_or_else(|| {
+ ErrorData::invalid_params(
+ format!(
+ "Access denied: token owner is not a member of workspace '{}'",
+ workspace_id
+ ),
+ None,
+ )
+ })
+ }
+
fn all_endpoint_tools(&self) -> Vec {
all_tools()
}
@@ -401,37 +462,67 @@ pub async fn add_www_authenticate_header(
}
}
-/// Middleware for gateway: extract workspace_id from the Bearer token in the DB
-/// and inject it as WorkspaceId extension so the MCP runner can use it.
+/// Extract the bearer token from either the `Authorization` header or the
+/// `?token=` query parameter (MCP clients commonly pass it in the URL).
+fn extract_gateway_token(request: &Request) -> Option {
+ if let Some(token) = request
+ .headers()
+ .get(axum::http::header::AUTHORIZATION)
+ .and_then(|v| v.to_str().ok())
+ .and_then(|h| h.strip_prefix("Bearer "))
+ {
+ return Some(token.to_string());
+ }
+ request.uri().query().and_then(|q| {
+ url::form_urlencoded::parse(q.as_bytes())
+ .find(|(k, _)| k == "token")
+ .map(|(_, v)| v.into_owned())
+ })
+}
+
+/// Middleware for gateway: resolve the MCP session mode from the Bearer token in
+/// the DB. A token bound to a workspace injects `WorkspaceId` (single-workspace
+/// mode). A workspace-less MCP token (`workspace_id IS NULL` with an `mcp:` scope)
+/// injects `MultiWorkspaceMcp` + `McpToken`, putting the runner in
+/// multi-workspace mode where tools take an explicit `workspace_id` argument.
pub async fn extract_workspace_from_token(
Extension(db): Extension,
mut request: Request,
next: Next,
) -> Response {
- if let Some(auth_header) = request
- .headers()
- .get(axum::http::header::AUTHORIZATION)
- .and_then(|v| v.to_str().ok())
- {
- if let Some(token) = auth_header.strip_prefix("Bearer ") {
- let t_hash = hash_token(token);
- match sqlx::query_scalar!(
- "SELECT workspace_id FROM token WHERE token_hash = $1 AND workspace_id IS NOT NULL AND (expiration > NOW() OR expiration IS NULL)",
- t_hash
- )
- .fetch_optional(&db)
- .await
- {
- Ok(Some(Some(workspace_id))) => {
+ if let Some(token) = extract_gateway_token(&request) {
+ let t_hash = hash_token(&token);
+ match sqlx::query!(
+ "SELECT workspace_id, scopes FROM token WHERE token_hash = $1 AND (expiration > NOW() OR expiration IS NULL)",
+ t_hash
+ )
+ .fetch_optional(&db)
+ .await
+ {
+ Ok(Some(row)) => match row.workspace_id {
+ Some(workspace_id) => {
request
.extensions_mut()
.insert(GatewayWorkspaceId(workspace_id.clone()));
request.extensions_mut().insert(WorkspaceId(workspace_id));
}
- Ok(_) => {}
- Err(e) => {
- tracing::error!("Gateway token workspace lookup failed: {}", e);
+ None => {
+ // Only enter multi-workspace mode for genuine MCP tokens; a
+ // full-privilege global token without mcp scope is rejected
+ // by the runner's mcp-scope check anyway.
+ let is_mcp = row
+ .scopes
+ .as_deref()
+ .is_some_and(|s| s.iter().any(|scope| scope.starts_with("mcp:")));
+ if is_mcp {
+ request.extensions_mut().insert(MultiWorkspaceMcp);
+ request.extensions_mut().insert(McpToken(token));
+ }
}
+ },
+ Ok(None) => {}
+ Err(e) => {
+ tracing::error!("Gateway token workspace lookup failed: {}", e);
}
}
}
@@ -472,11 +563,12 @@ pub async fn setup_mcp_server(
db: DB,
user_db: UserDB,
base_internal_url: String,
+ auth_cache: Arc,
) -> anyhow::Result<(Router, CancellationToken)> {
let cancellation_token = CancellationToken::new();
let session_manager = Arc::new(LocalSessionManager::default());
- let backend = WindmillBackend::new(db, user_db, base_internal_url);
+ let backend = WindmillBackend::new(db, user_db, base_internal_url, auth_cache);
let runner = Runner::new(backend);
let service_config = StreamableHttpServerConfig {
diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs
index 2184c1548f..ba4ea391d5 100644
--- a/backend/windmill-api/src/mcp/utils.rs
+++ b/backend/windmill-api/src/mcp/utils.rs
@@ -412,12 +412,50 @@ pub fn build_request_body(
args_map: &serde_json::Map,
body_schema: &Option,
body_field_renames: &Option,
+ path_params_schema: &Option,
+ query_params_schema: &Option,
) -> Option {
if method == "GET" {
return None;
}
let schema = body_schema.as_ref()?;
+
+ let has_declared_props = schema
+ .get("properties")
+ .and_then(|p| p.as_object())
+ .map(|o| !o.is_empty())
+ .unwrap_or(false);
+
+ // Pass-through body: the schema declares no explicit properties (e.g.
+ // runScriptByPath / runFlowByPath, whose body is `additionalProperties: true`
+ // and carries the script/flow arguments verbatim). Forward every argument
+ // that isn't already consumed by a path or query parameter — without this the
+ // request body would be empty and parameterized runs would lose their args.
+ if !has_declared_props {
+ if schema.get("type").and_then(|t| t.as_str()) != Some("object") {
+ return None;
+ }
+ let consumed: std::collections::HashSet<&str> = [path_params_schema, query_params_schema]
+ .into_iter()
+ .filter_map(|s| s.as_ref())
+ .filter_map(|s| s.get("properties").and_then(|p| p.as_object()))
+ .flat_map(|props| props.keys().map(|k| k.as_str()))
+ .collect();
+
+ let body_map: serde_json::Map = args_map
+ .iter()
+ .filter(|(k, v)| !consumed.contains(k.as_str()) && !v.is_null())
+ .map(|(k, v)| (k.clone(), v.clone()))
+ .collect();
+
+ return if body_map.is_empty() {
+ None
+ } else {
+ Some(Value::Object(body_map))
+ };
+ }
+
let props = schema.get("properties")?.as_object()?;
let body_map: serde_json::Map = props
@@ -540,6 +578,65 @@ mod tests {
use super::*;
use serde_json::json;
+ #[test]
+ fn build_request_body_passthrough_forwards_script_args_minus_path() {
+ // runScriptByPath-shaped body: additionalProperties, no declared props.
+ // `path` is a path param and must be excluded; the rest are the script's
+ // arguments and must be forwarded verbatim.
+ let body_schema = Some(json!({ "type": "object", "additionalProperties": true }));
+ let path_schema = Some(json!({
+ "type": "object",
+ "properties": { "path": { "type": "string" } },
+ "required": ["path"]
+ }));
+ let args: serde_json::Map = json!({
+ "path": "u/admin/my_script",
+ "name": "alice",
+ "count": 3
+ })
+ .as_object()
+ .unwrap()
+ .clone();
+
+ let body = build_request_body("POST", &args, &body_schema, &None, &path_schema, &None)
+ .expect("passthrough body should be built");
+ let obj = body.as_object().unwrap();
+ assert_eq!(obj.get("name"), Some(&json!("alice")));
+ assert_eq!(obj.get("count"), Some(&json!(3)));
+ assert!(
+ !obj.contains_key("path"),
+ "path param must be excluded from body"
+ );
+ }
+
+ #[test]
+ fn build_request_body_declared_props_only_forwards_declared() {
+ // Endpoints with explicit properties keep the strict declared-only behavior.
+ let body_schema = Some(json!({
+ "type": "object",
+ "properties": { "value": { "type": "string" } },
+ "required": ["value"]
+ }));
+ let args: serde_json::Map = json!({ "value": "x", "sneaky": "y" })
+ .as_object()
+ .unwrap()
+ .clone();
+ let body = build_request_body("POST", &args, &body_schema, &None, &None, &None).unwrap();
+ let obj = body.as_object().unwrap();
+ assert_eq!(obj.get("value"), Some(&json!("x")));
+ assert!(
+ !obj.contains_key("sneaky"),
+ "undeclared args must be dropped"
+ );
+ }
+
+ #[test]
+ fn build_request_body_get_has_no_body() {
+ let body_schema = Some(json!({ "type": "object", "additionalProperties": true }));
+ let args: serde_json::Map = json!({ "a": 1 }).as_object().unwrap().clone();
+ assert!(build_request_body("GET", &args, &body_schema, &None, &None, &None).is_none());
+ }
+
#[test]
fn validate_path_param_value_accepts_legitimate_windmill_paths() {
for ok in [
diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs
index 4d1d48a53d..0ee326ce6b 100644
--- a/backend/windmill-common/src/global_settings.rs
+++ b/backend/windmill-common/src/global_settings.rs
@@ -14,6 +14,12 @@ pub const WS_BASE_URL_SETTING: &str = "ws_base_url";
pub const OAUTH_SETTING: &str = "oauths";
pub const AI_CONFIG_SETTING: &str = "ai_config";
pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
+pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides";
+/// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor
+/// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded
+/// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob).
+/// Enforced at write time and defensively on load.
+pub const MAX_RETENTION_OVERRIDE_WORKSPACES: usize = 10;
pub const AUDIT_LOG_RETENTION_DAYS_SETTING: &str = "audit_log_retention_days";
pub const STORE_AUDIT_LOGS_S3_SETTING: &str = "store_audit_logs_s3";
/// `background_task_state.name` for the audit-log → object-store export cursor.
diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs
index acf97734d8..8f3dae3203 100644
--- a/backend/windmill-common/src/lib.rs
+++ b/backend/windmill-common/src/lib.rs
@@ -260,6 +260,16 @@ lazy_static::lazy_static! {
pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None);
pub static ref JOB_RETENTION_SECS: AtomicI64 = AtomicI64::new(0);
+ /// Per-workspace overrides of `JOB_RETENTION_SECS` (EE-only), keyed by workspace_id, in seconds.
+ /// Sourced from the `retention_period_secs_overrides` global setting and cached here so the
+ /// cleanup sweep reads it without a per-tick DB query. A workspace may be given a longer OR
+ /// shorter window than the instance-wide value; `0` means "keep forever" for that workspace.
+ pub static ref JOB_RETENTION_SECS_OVERRIDES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(std::collections::HashMap::new());
+ /// Whether `JOB_RETENTION_SECS_OVERRIDES` has ever been loaded successfully (a valid map, an
+ /// explicit unset, or CE's no-op). Until then the empty cache is "unknown, not confirmed empty",
+ /// so the retention sweep must NOT run globally — that would delete jobs a longer-retention
+ /// workspace configured before its override could be read.
+ pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false);
pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0);
pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false);
diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs
index 2f100d5ae1..627636b8f5 100644
--- a/backend/windmill-common/src/ssrf.rs
+++ b/backend/windmill-common/src/ssrf.rs
@@ -4,6 +4,8 @@ use crate::error::Error;
pub const ALLOW_PRIVATE_MCP_SERVER_URLS_ENV: &str = "ALLOW_PRIVATE_MCP_SERVER_URLS";
+pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METADATA_URLS";
+
/// Why a URL failed SSRF validation.
///
/// The distinction matters for callers that gate private endpoints behind a
@@ -124,6 +126,30 @@ pub fn allow_private_mcp_server_urls() -> bool {
.is_some_and(|v| v == "true" || v == "1")
}
+pub fn allow_private_saml_metadata_urls() -> bool {
+ std::env::var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV)
+ .ok()
+ .is_some_and(|v| v == "true" || v == "1")
+}
+
+pub async fn validate_saml_metadata_url(url: &str) -> Result<(), SsrfValidationError> {
+ let parsed =
+ url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?;
+
+ match parsed.scheme() {
+ "http" | "https" => {}
+ scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())),
+ }
+
+ parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
+
+ if allow_private_saml_metadata_urls() {
+ return Ok(());
+ }
+
+ validate_url_for_ssrf(url).await
+}
+
pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationError> {
let parsed =
url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?;
@@ -161,6 +187,16 @@ pub fn mcp_ssrf_error_message(e: &SsrfValidationError) -> String {
}
}
+pub fn saml_ssrf_error_message(e: &SsrfValidationError) -> String {
+ match e {
+ SsrfValidationError::Private { .. } => format!(
+ "{e}. If you need to use private/internal SAML metadata URLs, \
+ set the {ALLOW_PRIVATE_SAML_METADATA_URLS_ENV}=true environment variable"
+ ),
+ _ => e.to_string(),
+ }
+}
+
fn is_private_ip(ip: &IpAddr) -> bool {
match ip {
IpAddr::V4(ipv4) => is_private_ipv4(ipv4),
@@ -223,6 +259,30 @@ mod tests {
}
}
+ struct PrivateSamlMetadataUrlsEnvGuard {
+ previous: Option,
+ }
+
+ impl PrivateSamlMetadataUrlsEnvGuard {
+ fn set(value: Option<&str>) -> Self {
+ let previous = std::env::var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV).ok();
+ match value {
+ Some(value) => std::env::set_var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV, value),
+ None => std::env::remove_var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV),
+ }
+ Self { previous }
+ }
+ }
+
+ impl Drop for PrivateSamlMetadataUrlsEnvGuard {
+ fn drop(&mut self) {
+ match &self.previous {
+ Some(value) => std::env::set_var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV, value),
+ None => std::env::remove_var(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV),
+ }
+ }
+ }
+
#[test]
fn test_private_ipv4() {
assert!(is_private_ipv4(&"127.0.0.1".parse().unwrap()));
@@ -360,4 +420,87 @@ mod tests {
.unwrap_err();
assert!(!mcp_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_MCP_SERVER_URLS_ENV));
}
+
+ #[tokio::test]
+ async fn allow_private_saml_metadata_urls_defaults_to_false() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(None);
+ assert!(!allow_private_saml_metadata_urls());
+ }
+
+ #[tokio::test]
+ async fn allow_private_saml_metadata_urls_honors_true_and_one() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("true"));
+ assert!(allow_private_saml_metadata_urls());
+
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("1"));
+ assert!(allow_private_saml_metadata_urls());
+
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("false"));
+ assert!(!allow_private_saml_metadata_urls());
+ }
+
+ #[tokio::test]
+ async fn validate_saml_metadata_url_blocks_private_by_default() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(None);
+
+ assert!(matches!(
+ validate_saml_metadata_url("http://127.0.0.1/metadata").await,
+ Err(SsrfValidationError::Private { resolved: false })
+ ));
+ }
+
+ #[tokio::test]
+ async fn validate_saml_metadata_url_allows_private_when_env_is_enabled() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("true"));
+
+ assert!(validate_saml_metadata_url("http://127.0.0.1/metadata")
+ .await
+ .is_ok());
+ }
+
+ #[tokio::test]
+ async fn validate_saml_metadata_url_allows_private_when_env_is_one() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("1"));
+
+ assert!(validate_saml_metadata_url("http://10.0.0.1/metadata")
+ .await
+ .is_ok());
+ }
+
+ #[tokio::test]
+ async fn validate_saml_metadata_url_keeps_syntax_guards_when_private_urls_are_allowed() {
+ let _lock = TEST_ENV_LOCK.lock().await;
+ let _guard = PrivateSamlMetadataUrlsEnvGuard::set(Some("true"));
+
+ assert!(matches!(
+ validate_saml_metadata_url("ftp://example.com/metadata").await,
+ Err(SsrfValidationError::DisallowedScheme(_))
+ ));
+ assert!(matches!(
+ validate_saml_metadata_url("not-a-url").await,
+ Err(SsrfValidationError::InvalidUrl(_))
+ ));
+ }
+
+ #[tokio::test]
+ async fn saml_ssrf_error_message_includes_env_hint_only_for_private_urls() {
+ let private_error = validate_url_for_ssrf("http://127.0.0.1/metadata")
+ .await
+ .unwrap_err();
+ assert!(saml_ssrf_error_message(&private_error)
+ .contains("ALLOW_PRIVATE_SAML_METADATA_URLS=true"));
+
+ let invalid_error = validate_url_for_ssrf("ftp://example.com/metadata")
+ .await
+ .unwrap_err();
+ assert!(
+ !saml_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_SAML_METADATA_URLS_ENV)
+ );
+ }
}
diff --git a/backend/windmill-mcp/src/common/scope.rs b/backend/windmill-mcp/src/common/scope.rs
index f43095d740..418458d4fe 100644
--- a/backend/windmill-mcp/src/common/scope.rs
+++ b/backend/windmill-mcp/src/common/scope.rs
@@ -23,6 +23,26 @@ pub struct McpScopeConfig {
}
impl McpScopeConfig {
+ /// Whether the token grants access to *any* concrete resource of this type by
+ /// path. Used to decide whether to advertise the run-by-path tools in
+ /// multi-workspace mode (a `mcp:scripts:*`-only token should see
+ /// `runScriptByPath` even without an endpoint scope). `mcp:all` grants
+ /// everything; `mcp:favorites` does NOT — favorites are an enumerated set the
+ /// caller can only reach through the per-item tools, not by naming an
+ /// arbitrary path, so it grants nothing here (mirrors `is_allowed`, which
+ /// returns false for a favorites token).
+ pub fn has_any(&self, resource_type: &str) -> bool {
+ if self.all {
+ return true;
+ }
+ match resource_type {
+ "script" => !self.scripts.is_empty(),
+ "flow" => !self.flows.is_empty(),
+ "endpoint" => !self.endpoints.is_empty(),
+ _ => false,
+ }
+ }
+
/// Check if a resource is allowed based on its type and path
pub fn is_allowed(&self, resource_type: &str, path: &str) -> bool {
if self.all {
@@ -324,6 +344,29 @@ mod tests {
parse_mcp_scopes(&scopes.iter().map(|s| s.to_string()).collect::>()).unwrap()
}
+ #[test]
+ fn test_has_any() {
+ // mcp:all grants everything by path.
+ assert!(cfg(&["mcp:all"]).has_any("script"));
+
+ // mcp:favorites grants NO arbitrary-path access (favorites are reached
+ // via per-item tools, not by naming a path) — matches is_allowed.
+ let fav = cfg(&["mcp:favorites"]);
+ assert!(!fav.has_any("script"));
+ assert!(!fav.has_any("flow"));
+ assert!(!fav.is_allowed("script", "f/anything/x"));
+
+ // Granular: only the resource types with at least one pattern.
+ let scripts_only = cfg(&["mcp:scripts:f/team/*"]);
+ assert!(scripts_only.has_any("script"));
+ assert!(!scripts_only.has_any("flow"));
+ assert!(!scripts_only.has_any("endpoint"));
+
+ let endpoints_only = cfg(&["mcp:endpoints:runScriptByPath"]);
+ assert!(!endpoints_only.has_any("script"));
+ assert!(endpoints_only.has_any("endpoint"));
+ }
+
#[test]
fn test_contains_subset_and_widening() {
// mcp:all contains anything.
diff --git a/backend/windmill-mcp/src/common/types.rs b/backend/windmill-mcp/src/common/types.rs
index 6161ca7963..d6bd88a397 100644
--- a/backend/windmill-mcp/src/common/types.rs
+++ b/backend/windmill-mcp/src/common/types.rs
@@ -15,6 +15,27 @@ use sqlx::FromRow;
#[derive(Clone, Debug)]
pub struct WorkspaceId(pub String);
+/// Marker extension inserted by the gateway middleware when an MCP token has no
+/// bound workspace (`workspace_id IS NULL`). Signals the runner to operate in
+/// multi-workspace mode: tools take an explicit `workspace_id` argument and the
+/// per-workspace auth is resolved on demand from the raw token.
+#[derive(Clone, Debug)]
+pub struct MultiWorkspaceMcp;
+
+/// Raw bearer token wrapper for Axum extensions. In multi-workspace mode the
+/// runner needs the raw token to re-resolve auth for each requested workspace.
+#[derive(Clone, Debug)]
+pub struct McpToken(pub String);
+
+/// Summary of a workspace the caller can access, returned by the
+/// `list_workspaces` tool in multi-workspace mode.
+#[derive(Serialize, Debug, Clone)]
+#[cfg_attr(feature = "server", derive(FromRow))]
+pub struct WorkspaceInfo {
+ pub id: String,
+ pub name: String,
+}
+
/// Hub API response structure
#[derive(Serialize, Deserialize, Debug)]
pub struct HubResponse {
diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs
index 7a6e007c4d..96a2bed8df 100644
--- a/backend/windmill-mcp/src/server/backend.rs
+++ b/backend/windmill-mcp/src/server/backend.rs
@@ -9,7 +9,7 @@ use serde_json::Value;
use std::collections::HashMap;
use crate::common::types::{
- FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
+ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo,
};
use crate::server::endpoints::EndpointTool;
@@ -159,6 +159,27 @@ pub trait McpBackend: Send + Sync + Clone + 'static {
args: Value,
) -> BackendResult;
+ // ─────────────────────────────────────────────────────────────────
+ // Multi-workspace support
+ // ─────────────────────────────────────────────────────────────────
+
+ /// List the workspaces the caller (identified by `auth`) can access. Used by
+ /// the `list_workspaces` tool exposed in multi-workspace mode.
+ async fn list_accessible_workspaces(
+ &self,
+ auth: &Self::Auth,
+ ) -> BackendResult>;
+
+ /// Resolve a workspace-specific auth for `workspace_id` from the raw bearer
+ /// `token`. Returns an error if the token's owner is not a member of the
+ /// workspace. Used in multi-workspace mode to authorize per-workspace tool
+ /// calls (the base auth carries no workspace-specific permissions).
+ async fn resolve_workspace_auth(
+ &self,
+ token: &str,
+ workspace_id: &str,
+ ) -> BackendResult;
+
// ─────────────────────────────────────────────────────────────────
// Endpoint Tools
// ─────────────────────────────────────────────────────────────────
diff --git a/backend/windmill-mcp/src/server/endpoints.rs b/backend/windmill-mcp/src/server/endpoints.rs
index 373db36eb1..c3a50f9715 100644
--- a/backend/windmill-mcp/src/server/endpoints.rs
+++ b/backend/windmill-mcp/src/server/endpoints.rs
@@ -73,6 +73,86 @@ pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool {
}
}
+/// Convert an endpoint tool to an MCP tool for multi-workspace mode.
+///
+/// Endpoints whose path is workspace-scoped (`/w/{workspace}/...`) gain a
+/// required `workspace_id` argument — in multi-workspace mode there is no
+/// ambient workspace, so the caller must name the target workspace explicitly.
+/// Global endpoints (e.g. docs search) are returned unchanged.
+pub fn endpoint_tool_to_mcp_tool_multi(tool: &EndpointTool) -> Tool {
+ let mut mcp_tool = endpoint_tool_to_mcp_tool(tool);
+
+ if !tool.path.contains("{workspace}") {
+ return mcp_tool;
+ }
+
+ let mut schema = (*mcp_tool.input_schema).clone();
+
+ if let Some(props) = schema.get_mut("properties").and_then(|p| p.as_object_mut()) {
+ props.insert(
+ "workspace_id".to_string(),
+ serde_json::json!({
+ "type": "string",
+ "description": "Target workspace id (from list_workspaces)."
+ }),
+ );
+ }
+
+ match schema.get_mut("required").and_then(|r| r.as_array_mut()) {
+ Some(req) => {
+ if !req.iter().any(|v| v.as_str() == Some("workspace_id")) {
+ req.insert(0, serde_json::Value::String("workspace_id".to_string()));
+ }
+ }
+ None => {
+ schema.insert("required".to_string(), serde_json::json!(["workspace_id"]));
+ }
+ }
+
+ // Surface the requirement in the prose description too (the schema is
+ // authoritative, but some models/clients lean on the text). Kept terse — this
+ // repeats across every workspace-scoped tool in the list.
+ if let Some(desc) = mcp_tool.description.take() {
+ mcp_tool.description = Some(format!("{desc} Requires `workspace_id`.").into());
+ } else {
+ mcp_tool.description = Some("Requires `workspace_id`.".into());
+ }
+
+ mcp_tool.input_schema = Arc::new(schema);
+ mcp_tool
+}
+
+/// Build the synthetic `list_workspaces` tool exposed only in multi-workspace
+/// mode. It takes no arguments and returns the workspaces the token can access.
+pub fn list_workspaces_tool() -> Tool {
+ let schema = serde_json::json!({
+ "type": "object",
+ "properties": {},
+ "required": []
+ });
+
+ Tool {
+ name: Cow::Borrowed("list_workspaces"),
+ description: Some(
+ "List the Windmill workspaces this token can access. Use the returned workspace ids as the `workspace_id` argument of the other tools."
+ .into(),
+ ),
+ input_schema: Arc::new(schema.as_object().unwrap().clone()),
+ title: Some("List accessible workspaces".to_string()),
+ output_schema: None,
+ icons: None,
+ annotations: Some(ToolAnnotations {
+ title: Some("List accessible workspaces".to_string()),
+ read_only_hint: Some(true),
+ destructive_hint: Some(false),
+ idempotent_hint: Some(true),
+ open_world_hint: Some(false),
+ }),
+ meta: None,
+ execution: None,
+ }
+}
+
/// Create appropriate annotations for endpoint tools based on HTTP method
fn create_endpoint_annotations(tool: &EndpointTool) -> ToolAnnotations {
let method = tool.method.as_ref();
@@ -116,3 +196,119 @@ fn merge_schema_into(
}
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn tool(name: &'static str, path: &'static str) -> EndpointTool {
+ EndpointTool {
+ name: Cow::Borrowed(name),
+ description: Cow::Borrowed("desc"),
+ instructions: Cow::Borrowed(""),
+ path: Cow::Borrowed(path),
+ method: Cow::Borrowed("GET"),
+ path_params_schema: None,
+ query_params_schema: Some(serde_json::json!({
+ "type": "object",
+ "properties": { "starred_only": { "type": "boolean" } },
+ "required": []
+ })),
+ body_schema: None,
+ path_field_renames: None,
+ query_field_renames: None,
+ body_field_renames: None,
+ }
+ }
+
+ #[test]
+ fn multi_injects_required_workspace_id_for_workspaced_tool() {
+ let mcp =
+ endpoint_tool_to_mcp_tool_multi(&tool("listScripts", "/w/{workspace}/scripts/list"));
+ let props = mcp
+ .input_schema
+ .get("properties")
+ .unwrap()
+ .as_object()
+ .unwrap();
+ assert!(
+ props.contains_key("workspace_id"),
+ "workspace_id must be added as a property"
+ );
+ // pre-existing param is preserved
+ assert!(props.contains_key("starred_only"));
+ let required = mcp
+ .input_schema
+ .get("required")
+ .unwrap()
+ .as_array()
+ .unwrap();
+ assert!(
+ required.iter().any(|v| v.as_str() == Some("workspace_id")),
+ "workspace_id must be required"
+ );
+ assert!(
+ mcp.description
+ .as_deref()
+ .unwrap_or_default()
+ .contains("workspace_id"),
+ "description must mention the workspace_id requirement"
+ );
+ }
+
+ #[test]
+ fn multi_leaves_global_tool_unchanged() {
+ let global = tool("searchDocs", "/docs/search");
+ let plain = endpoint_tool_to_mcp_tool(&global);
+ let mcp = endpoint_tool_to_mcp_tool_multi(&global);
+ assert_eq!(
+ mcp.description, plain.description,
+ "global tool description must be unchanged"
+ );
+ let props = mcp
+ .input_schema
+ .get("properties")
+ .unwrap()
+ .as_object()
+ .unwrap();
+ assert!(
+ !props.contains_key("workspace_id"),
+ "global tools (no {{workspace}} in path) must not gain a workspace_id arg"
+ );
+ let required = mcp
+ .input_schema
+ .get("required")
+ .unwrap()
+ .as_array()
+ .unwrap();
+ assert!(required.iter().all(|v| v.as_str() != Some("workspace_id")));
+ }
+
+ #[test]
+ fn multi_does_not_duplicate_workspace_id() {
+ // Even if run twice, workspace_id stays a single required entry.
+ let once = endpoint_tool_to_mcp_tool_multi(&tool("listFlows", "/w/{workspace}/flows/list"));
+ let required = once
+ .input_schema
+ .get("required")
+ .unwrap()
+ .as_array()
+ .unwrap();
+ let count = required
+ .iter()
+ .filter(|v| v.as_str() == Some("workspace_id"))
+ .count();
+ assert_eq!(
+ count, 1,
+ "workspace_id must appear exactly once in required"
+ );
+ }
+
+ #[test]
+ fn list_workspaces_tool_has_no_params() {
+ let t = list_workspaces_tool();
+ assert_eq!(t.name.as_ref(), "list_workspaces");
+ let required = t.input_schema.get("required").unwrap().as_array().unwrap();
+ assert!(required.is_empty());
+ }
+}
diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs
index 688c12b827..d5d49bc746 100644
--- a/backend/windmill-mcp/src/server/mod.rs
+++ b/backend/windmill-mcp/src/server/mod.rs
@@ -11,8 +11,12 @@ pub mod runner;
pub mod tools;
// Re-export main types
+pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo};
pub use backend::{BackendResult, McpAuth, McpBackend};
-pub use endpoints::{endpoint_tool_to_mcp_tool, is_endpoint_read_only, EndpointTool};
+pub use endpoints::{
+ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only,
+ list_workspaces_tool, EndpointTool,
+};
pub use runner::Runner;
pub use tools::create_tool_from_item;
diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs
index 6d33573d95..dd810269d9 100644
--- a/backend/windmill-mcp/src/server/runner.rs
+++ b/backend/windmill-mcp/src/server/runner.rs
@@ -9,9 +9,11 @@ use crate::common::transform::{
extract_hub_version_id_from_hashed, extract_path_prefix_from_hashed, parse_tool_prefix,
reverse_transform, reverse_transform_key,
};
-use crate::common::types::{ResourceInfo, ToolableItem, WorkspaceId};
+use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId};
use crate::server::backend::{McpAuth, McpBackend};
-use crate::server::endpoints::endpoint_tool_to_mcp_tool;
+use crate::server::endpoints::{
+ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool,
+};
use crate::server::tools::create_tool_from_item;
use rmcp::handler::server::ServerHandler;
use rmcp::model::{
@@ -61,16 +63,28 @@ impl Clone for Runner {
}
}
+/// Whether the request targets one bound workspace or spans every workspace the
+/// token can access.
+enum McpMode {
+ /// A single workspace, resolved from the URL path or the token's bound
+ /// workspace. Tools operate against this workspace implicitly.
+ Single(String),
+ /// The token has no bound workspace (`workspace_id IS NULL`). Tools take an
+ /// explicit `workspace_id` argument; the wrapped value is the raw bearer
+ /// token, used to re-resolve auth per requested workspace.
+ Multi(String),
+}
+
impl Runner {
/// Create a new Runner with the given backend
pub fn new(backend: B) -> Self {
Self { backend: Arc::new(backend) }
}
- /// Extract authentication and workspace from request context
+ /// Extract authentication and the workspace mode from request context
fn extract_context(
context: &RequestContext,
- ) -> Result<(B::Auth, String), ErrorData> {
+ ) -> Result<(B::Auth, McpMode), ErrorData> {
let http_parts = context.extensions.get::().ok_or_else(|| {
tracing::error!("http::request::Parts not found");
ErrorData::internal_error("http::request::Parts not found", None)
@@ -81,15 +95,6 @@ impl Runner {
ErrorData::internal_error("Auth extension not found", None)
})?;
- let workspace_id = http_parts
- .extensions
- .get::()
- .ok_or_else(|| {
- tracing::error!("WorkspaceId not found");
- ErrorData::internal_error("WorkspaceId not found", None)
- })
- .map(|w_id| w_id.0.clone())?;
-
// Validate MCP scope
if !auth.has_mcp_scope() {
tracing::error!("Unauthorized: missing mcp scope");
@@ -99,7 +104,39 @@ impl Runner {
));
}
- Ok((auth.clone(), workspace_id))
+ let mode = if http_parts.extensions.get::().is_some() {
+ let token = http_parts.extensions.get::().ok_or_else(|| {
+ tracing::error!("MultiWorkspaceMcp set but McpToken missing");
+ ErrorData::internal_error("MCP token not found for multi-workspace session", None)
+ })?;
+ McpMode::Multi(token.0.clone())
+ } else {
+ let workspace_id = http_parts
+ .extensions
+ .get::()
+ .ok_or_else(|| {
+ tracing::error!("WorkspaceId not found");
+ ErrorData::internal_error("WorkspaceId not found", None)
+ })
+ .map(|w_id| w_id.0.clone())?;
+ McpMode::Single(workspace_id)
+ };
+
+ Ok((auth.clone(), mode))
+ }
+}
+
+/// The run-by-path endpoint tools execute an arbitrary script/flow named by a
+/// `path` argument. In multi-workspace mode they are the only way to run
+/// scripts/flows, so their authorization must honor the `mcp:scripts:` /
+/// `mcp:flows:` path scopes (not the generic endpoint scope) — otherwise a
+/// granular token could run items outside its allowed paths. Returns the scope
+/// resource type ("script"/"flow") for these endpoints, `None` otherwise.
+fn run_by_path_scope_kind(endpoint_name: &str) -> Option<&'static str> {
+ match endpoint_name {
+ "runScriptByPath" => Some("script"),
+ "runFlowByPath" => Some("flow"),
+ _ => None,
}
}
@@ -137,16 +174,99 @@ impl ServerHandler for Runner {
_request: Option,
context: RequestContext,
) -> Result {
- let (auth, workspace_id) = Self::extract_context(&context)?;
+ let (auth, mode) = Self::extract_context(&context)?;
// Parse MCP scopes to determine what to expose
let scopes = auth.scopes().unwrap_or(&[]);
let scope_config =
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
- let favorites_only = scope_config.favorites;
let read_only = auth.read_only();
+ match mode {
+ McpMode::Single(workspace_id) => {
+ self.list_tools_single(&auth, &workspace_id, &scope_config, read_only)
+ .await
+ }
+ // Multi-workspace: expose the generic endpoint tools (each taking an
+ // explicit workspace_id) plus list_workspaces. Per-workspace scripts
+ // and flows are intentionally not enumerated here — doing so across
+ // every workspace would overload the tool list; callers run them via
+ // runScriptByPath / runFlowByPath with a workspace_id instead.
+ McpMode::Multi(_) => Ok(self.list_tools_multi(&scope_config, read_only)),
+ }
+ }
+
+ async fn call_tool(
+ &self,
+ request: CallToolRequestParams,
+ context: RequestContext,
+ ) -> Result {
+ let (auth, mode) = Self::extract_context(&context)?;
+
+ // Parse MCP scopes for authorization
+ let scopes = auth.scopes().unwrap_or(&[]);
+ let scope_config =
+ parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
+ let read_only = auth.read_only();
+
+ let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
+
+ match mode {
+ McpMode::Single(workspace_id) => {
+ self.call_tool_single(
+ &auth,
+ &workspace_id,
+ &scope_config,
+ read_only,
+ request.name,
+ args,
+ )
+ .await
+ }
+ McpMode::Multi(token) => {
+ self.call_tool_multi(&auth, &token, &scope_config, read_only, request.name, args)
+ .await
+ }
+ }
+ }
+
+ async fn list_resources(
+ &self,
+ _request: Option,
+ _context: RequestContext,
+ ) -> Result {
+ Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None })
+ }
+
+ async fn list_prompts(
+ &self,
+ _request: Option,
+ _context: RequestContext,
+ ) -> Result {
+ Ok(ListPromptsResult::default())
+ }
+
+ async fn list_resource_templates(
+ &self,
+ _request: Option,
+ _context: RequestContext,
+ ) -> Result {
+ Ok(ListResourceTemplatesResult::default())
+ }
+}
+
+impl Runner {
+ /// List tools for a single, bound workspace (URL-path or token-bound).
+ async fn list_tools_single(
+ &self,
+ auth: &B::Auth,
+ workspace_id: &str,
+ scope_config: &crate::common::scope::McpScopeConfig,
+ read_only: bool,
+ ) -> Result {
+ let favorites_only = scope_config.favorites;
+
let mut tools = Vec::new();
// Read-only tokens cannot run scripts/flows/hub-scripts (running is a
@@ -155,10 +275,10 @@ impl ServerHandler for Runner {
if !read_only {
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
self.backend
- .list_scripts(&auth, &workspace_id, favorites_only, None),
+ .list_scripts(auth, workspace_id, favorites_only, None),
self.backend
- .list_flows(&auth, &workspace_id, favorites_only, None),
- self.backend.list_resource_types(&auth, &workspace_id),
+ .list_flows(auth, workspace_id, favorites_only, None),
+ self.backend.list_resource_types(auth, workspace_id),
async {
if let Some(ref apps) = scope_config.hub_apps {
self.backend.list_hub_scripts(Some(apps)).await
@@ -199,7 +319,7 @@ impl ServerHandler for Runner {
.map(|rt| {
let backend = self.backend.clone();
let auth = auth.clone();
- let workspace_id = workspace_id.clone();
+ let workspace_id = workspace_id.to_string();
async move {
backend
.list_resources(&auth, &workspace_id, &rt)
@@ -257,25 +377,20 @@ impl ServerHandler for Runner {
Ok(ListToolsResult { tools, next_cursor: None, meta: None })
}
- async fn call_tool(
+ /// Handle a tool call for a single, bound workspace.
+ async fn call_tool_single(
&self,
- request: CallToolRequestParams,
- context: RequestContext,
+ auth: &B::Auth,
+ workspace_id: &str,
+ scope_config: &crate::common::scope::McpScopeConfig,
+ read_only: bool,
+ name: std::borrow::Cow<'static, str>,
+ args: Value,
) -> Result {
- let (auth, workspace_id) = Self::extract_context(&context)?;
-
- // Parse MCP scopes for authorization
- let scopes = auth.scopes().unwrap_or(&[]);
- let scope_config =
- parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
- let read_only = auth.read_only();
-
- let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
-
// Check if this is an endpoint tool
let endpoint_tools = self.backend.all_endpoint_tools();
for endpoint_tool in &endpoint_tools {
- if endpoint_tool.name.as_ref() == request.name {
+ if endpoint_tool.name.as_ref() == name.as_ref() {
// Validate endpoint scope
if scope_config.granular
&& !scope_config.is_allowed("endpoint", &endpoint_tool.name)
@@ -301,7 +416,7 @@ impl ServerHandler for Runner {
// This is an endpoint tool, call via backend
let result = self
.backend
- .call_endpoint(&auth, &workspace_id, endpoint_tool, args)
+ .call_endpoint(auth, workspace_id, endpoint_tool, args)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?;
@@ -319,53 +434,50 @@ impl ServerHandler for Runner {
return Err(ErrorData::internal_error(
format!(
"Access denied: tool '{}' runs a script/flow and this token is restricted to read-only operations",
- request.name
+ name
),
None,
));
}
// Resolve the tool name to (type, path, is_hub)
- let (type_str, is_hub, is_hashed) = parse_tool_prefix(&request.name).map_err(|e| {
+ let (type_str, is_hub, is_hashed) = parse_tool_prefix(name.as_ref()).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?;
let (tool_type, path, is_hub) = if !is_hashed {
- reverse_transform(&request.name).map_err(|e| {
+ reverse_transform(name.as_ref()).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
})?
} else if is_hub {
- let version_id = extract_hub_version_id_from_hashed(&request.name).map_err(|e| {
+ let version_id = extract_hub_version_id_from_hashed(name.as_ref()).map_err(|e| {
ErrorData::internal_error(format!("Failed to extract hub version_id: {}", e), None)
})?;
(type_str, version_id, true)
} else {
- let path_prefix = extract_path_prefix_from_hashed(&request.name);
+ let path_prefix = extract_path_prefix_from_hashed(name.as_ref());
let favorites_only = scope_config.favorites;
let matched_path = if type_str == "script" {
find_matching_path(
self.backend
- .list_scripts(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
+ .list_scripts(auth, workspace_id, favorites_only, path_prefix.as_deref())
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?,
- &request.name,
+ name.as_ref(),
)
} else {
find_matching_path(
self.backend
- .list_flows(&auth, &workspace_id, favorites_only, path_prefix.as_deref())
+ .list_flows(auth, workspace_id, favorites_only, path_prefix.as_deref())
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?,
- &request.name,
+ name.as_ref(),
)
};
let matched_path = matched_path.ok_or_else(|| {
ErrorData::internal_error(
- format!(
- "No {} found matching hashed tool name '{}'",
- type_str, request.name
- ),
+ format!("No {} found matching hashed tool name '{}'", type_str, name),
None,
)
})?;
@@ -396,7 +508,7 @@ impl ServerHandler for Runner {
.map_err(|e| ErrorData::internal_error(e.message, None))?
} else {
self.backend
- .get_item_schema(&auth, &workspace_id, &path, tool_type)
+ .get_item_schema(auth, workspace_id, &path, tool_type)
.await
.map_err(|e| ErrorData::internal_error(e.message, None))?
};
@@ -422,11 +534,11 @@ impl ServerHandler for Runner {
// Execute script or flow
let result = if tool_type == "script" {
self.backend
- .run_script(&auth, &workspace_id, &script_or_flow_path, transformed_args)
+ .run_script(auth, workspace_id, &script_or_flow_path, transformed_args)
.await
} else {
self.backend
- .run_flow(&auth, &workspace_id, &script_or_flow_path, transformed_args)
+ .run_flow(auth, workspace_id, &script_or_flow_path, transformed_args)
.await
};
@@ -443,27 +555,181 @@ impl ServerHandler for Runner {
}
}
- async fn list_resources(
+ /// List tools for a multi-workspace session: the synthetic `list_workspaces`
+ /// tool plus every generic endpoint tool, each taking an explicit
+ /// `workspace_id` argument.
+ fn list_tools_multi(
&self,
- _request: Option,
- _context: RequestContext,
- ) -> Result {
- Ok(ListResourcesResult { resources: vec![], next_cursor: None, meta: None })
+ scope_config: &crate::common::scope::McpScopeConfig,
+ read_only: bool,
+ ) -> ListToolsResult {
+ let mut tools = vec![list_workspaces_tool()];
+
+ let endpoint_tools = self.backend.all_endpoint_tools();
+ for endpoint_tool in endpoint_tools {
+ // Run-by-path tools are gated by script/flow scope (they run an
+ // arbitrary path); every other endpoint by the endpoint scope.
+ let allowed = match run_by_path_scope_kind(&endpoint_tool.name) {
+ Some(kind) => scope_config.has_any(kind),
+ None => {
+ !scope_config.granular
+ || scope_config.is_allowed("endpoint", &endpoint_tool.name)
+ }
+ };
+ if !allowed {
+ continue;
+ }
+ if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) {
+ continue;
+ }
+
+ tools.push(endpoint_tool_to_mcp_tool_multi(&endpoint_tool));
+ }
+
+ ListToolsResult { tools, next_cursor: None, meta: None }
}
- async fn list_prompts(
+ /// Handle a tool call for a multi-workspace session. `base_auth` is the
+ /// workspace-less identity derived from the token; per-workspace auth is
+ /// resolved on demand from `token` for the workspace named in the args.
+ async fn call_tool_multi(
&self,
- _request: Option,
- _context: RequestContext,
- ) -> Result {
- Ok(ListPromptsResult::default())
- }
+ base_auth: &B::Auth,
+ token: &str,
+ scope_config: &crate::common::scope::McpScopeConfig,
+ read_only: bool,
+ name: std::borrow::Cow<'static, str>,
+ args: Value,
+ ) -> Result {
+ if name.as_ref() == "list_workspaces" {
+ let workspaces = self
+ .backend
+ .list_accessible_workspaces(base_auth)
+ .await
+ .map_err(|e| ErrorData::internal_error(e.message, None))?;
+ return Ok(CallToolResult::success(vec![Content::text(
+ serde_json::to_string_pretty(&workspaces).unwrap_or_else(|_| "[]".to_string()),
+ )]));
+ }
- async fn list_resource_templates(
- &self,
- _request: Option,
- _context: RequestContext,
- ) -> Result {
- Ok(ListResourceTemplatesResult::default())
+ // Only endpoint tools are exposed in multi-workspace mode; scripts and
+ // flows are run through the runScriptByPath / runFlowByPath endpoints.
+ let endpoint_tools = self.backend.all_endpoint_tools();
+ let endpoint_tool = endpoint_tools
+ .iter()
+ .find(|t| t.name.as_ref() == name.as_ref())
+ .ok_or_else(|| {
+ ErrorData::invalid_params(
+ format!(
+ "Unknown tool '{}' in multi-workspace mode. Available tools are list_workspaces and the generic API endpoint tools (run scripts/flows via runScriptByPath / runFlowByPath).",
+ name
+ ),
+ None,
+ )
+ })?;
+
+ // Authorize the tool. Run-by-path endpoints (runScriptByPath /
+ // runFlowByPath) run an arbitrary `path` and must be checked against the
+ // script/flow scope for that path — the endpoint scope alone would let a
+ // granular token run items outside its allowed paths.
+ match run_by_path_scope_kind(&endpoint_tool.name) {
+ Some(kind) => {
+ let path = args
+ .get("path")
+ .and_then(|v| v.as_str())
+ .filter(|s| !s.is_empty())
+ .ok_or_else(|| {
+ ErrorData::invalid_params(
+ format!(
+ "Missing required 'path' argument for tool '{}'.",
+ endpoint_tool.name
+ ),
+ None,
+ )
+ })?;
+ // No `granular` gate: is_allowed already encodes every mode —
+ // true for mcp:all, pattern-matched for granular scopes, and
+ // false for mcp:favorites (a favorites token can't run an
+ // arbitrary path, only its enumerated favorites).
+ if !scope_config.is_allowed(kind, path) {
+ return Err(ErrorData::internal_error(
+ format!("Access denied: {} '{}' not in token scope", kind, path),
+ None,
+ ));
+ }
+ }
+ None => {
+ if scope_config.granular
+ && !scope_config.is_allowed("endpoint", &endpoint_tool.name)
+ {
+ return Err(ErrorData::internal_error(
+ format!(
+ "Access denied: endpoint '{}' not in token scope",
+ endpoint_tool.name
+ ),
+ None,
+ ));
+ }
+ }
+ }
+ if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) {
+ return Err(ErrorData::internal_error(
+ format!(
+ "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations",
+ endpoint_tool.name
+ ),
+ None,
+ ));
+ }
+
+ // Workspace-scoped endpoints need an explicit target workspace and a
+ // per-workspace auth; global endpoints (e.g. docs) use the base identity.
+ let needs_workspace = endpoint_tool.path.contains("{workspace}");
+ let (workspace_id, resolved_auth) = if needs_workspace {
+ let workspace_id = args
+ .get("workspace_id")
+ .and_then(|v| v.as_str())
+ .filter(|s| !s.is_empty())
+ .ok_or_else(|| {
+ ErrorData::invalid_params(
+ format!(
+ "Missing required 'workspace_id' argument for tool '{}'. Call list_workspaces to see the workspaces you can access.",
+ endpoint_tool.name
+ ),
+ None,
+ )
+ })?
+ .to_string();
+
+ let resolved = self
+ .backend
+ .resolve_workspace_auth(token, &workspace_id)
+ .await
+ .map_err(|e| ErrorData::internal_error(e.message, None))?;
+ (workspace_id, resolved)
+ } else {
+ (String::new(), base_auth.clone())
+ };
+
+ // `workspace_id` is a synthetic argument only this layer understands; the
+ // target workspace is passed to call_endpoint separately. Strip it so it
+ // can't leak into a pass-through request body (e.g. runScriptByPath, whose
+ // body forwards all remaining args as the script's arguments).
+ let mut args = args;
+ if let Value::Object(map) = &mut args {
+ map.remove("workspace_id");
+ }
+
+ let result = self
+ .backend
+ .call_endpoint(&resolved_auth, &workspace_id, endpoint_tool, args)
+ .await
+ .map_err(|e| ErrorData::internal_error(e.message, None))?;
+
+ Ok(CallToolResult::success(vec![Content::text(
+ truncate_tool_result(
+ serde_json::to_string_pretty(&result).unwrap_or_else(|_| "{}".to_string()),
+ ),
+ )]))
}
}
diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs
index 99f33da3bf..4ce45eb8c0 100644
--- a/backend/windmill-queue/src/jobs.rs
+++ b/backend/windmill-queue/src/jobs.rs
@@ -921,8 +921,8 @@ lazy_static::lazy_static! {
pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
pub static ref MAX_RESULT_SIZE_MB: usize = std::env::var("MAX_RESULT_SIZE_MB").unwrap_or("500".to_string()).parse().unwrap_or(500);
- // Cache for restart_unless_cancelled flag - keyed by (hash, workspace_id)
- static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), bool> = Cache::new(10000);
+ // Cache for perpetual-restart settings (restart_unless_cancelled, timeout) - keyed by (hash, workspace_id)
+ static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), (bool, Option)> = Cache::new(10000);
// Cache for workspace error handler settings with 60s TTL
// Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, expiry_timestamp)
@@ -1538,21 +1538,27 @@ async fn restart_job_if_perpetual_inner(
) -> Result<(), Error> {
let cache_key = (hash.0, queued_job.workspace_id.clone());
- let restart = if let Some(cached) = RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key) {
+ let (restart, script_timeout) = if let Some(cached) =
+ RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key)
+ {
cached
} else {
- let restart = sqlx::query_scalar!(
- "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2",
+ let row = sqlx::query!(
+ "SELECT restart_unless_cancelled, timeout FROM script WHERE hash = $1 AND workspace_id = $2",
hash.0,
&queued_job.workspace_id
)
.fetch_optional(db)
- .await?
- .flatten()
- .unwrap_or(false);
+ .await?;
- RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, restart);
- restart
+ let restart = row
+ .as_ref()
+ .and_then(|r| r.restart_unless_cancelled)
+ .unwrap_or(false);
+ let script_timeout = row.and_then(|r| r.timeout);
+
+ RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, (restart, script_timeout));
+ (restart, script_timeout)
};
if restart {
@@ -1623,7 +1629,7 @@ async fn restart_job_if_perpetual_inner(
None,
true,
Some(queued_job.tag.clone()),
- None,
+ script_timeout,
None,
queued_job.priority,
None,
diff --git a/backend/windmill-trigger/src/global_handler.rs b/backend/windmill-trigger/src/global_handler.rs
index 98b689dda9..00727eac68 100644
--- a/backend/windmill-trigger/src/global_handler.rs
+++ b/backend/windmill-trigger/src/global_handler.rs
@@ -36,8 +36,11 @@ async fn get_suspended_trigger(
trigger_kind: &JobTriggerKind,
path: &str,
) -> Result {
+ // Only trigger kinds backed by a `_trigger` table support reassignment.
+ // `app` (and webhook/schedule) have no such table, so reject them with a clear
+ // error instead of failing on a missing-relation database error below.
match trigger_kind {
- JobTriggerKind::Webhook | JobTriggerKind::Schedule => {
+ JobTriggerKind::Webhook | JobTriggerKind::Schedule | JobTriggerKind::App => {
return Err(Error::BadRequest(format!(
"{} triggers do not support job reassignment",
trigger_kind
diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs
index f9372b949d..95ac69bc2c 100644
--- a/backend/windmill-types/src/jobs.rs
+++ b/backend/windmill-types/src/jobs.rs
@@ -49,6 +49,11 @@ pub enum JobTriggerKind {
// A run pushed by the pipeline freshness watchdog (EE) because the
// script's `// freshness` window elapsed without a successful run.
Freshness,
+ // A run launched by a deployed app's runtime (`execute_component`). `trigger`
+ // carries the app path. This is the authoritative app-origination marker: a
+ // direct `/jobs/run` cannot set it, so it distinguishes files an app actually
+ // produced from files a viewer forged by running a declared runnable directly.
+ App,
}
impl std::fmt::Display for JobTriggerKind {
@@ -72,6 +77,7 @@ impl std::fmt::Display for JobTriggerKind {
JobTriggerKind::CiTest => "ci_test",
JobTriggerKind::Asset => "asset",
JobTriggerKind::Freshness => "freshness",
+ JobTriggerKind::App => "app",
};
write!(f, "{}", kind)
}
diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs
index a446548e1f..2c5c17bb27 100644
--- a/backend/windmill-worker/src/deno_executor.rs
+++ b/backend/windmill-worker/src/deno_executor.rs
@@ -481,7 +481,15 @@ try {{
args.push("--allow-write=./");
args.push("--allow-env");
args.push("--allow-import");
- args.push("--allow-run=git,/usr/bin/chromium");
+ // Deliberately NO --allow-run: unlike every other language, deno jobs
+ // are never nsjail-wrapped, so the Deno permission model is the *only*
+ // sandbox boundary. Any allowed binary that can spawn a subprocess
+ // therefore escapes it entirely — git via hook configs
+ // (`-c core.fsmonitor=`) and chromium via subprocess-launcher flags
+ // (`--renderer-cmd-prefix` / `--gpu-launcher`) both coerce /bin/sh and
+ // defeat the guarantee (GHSA-gj6h-vw66-mr8f). Omitting the flag denies
+ // all subprocess execution. Admins who accept the risk (e.g. puppeteer)
+ // can re-add specific binaries via DENO_FLAGS.
} else {
args.push("-A");
}
diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs
index 26fbdb21af..f2f36621a5 100644
--- a/backend/windmill-worker/src/result_processor.rs
+++ b/backend/windmill-worker/src/result_processor.rs
@@ -300,6 +300,12 @@ pub fn start_background_processor(
worker_name: String,
killpill_tx: KillpillSender,
is_dedicated_worker: bool,
+ // True when this processor runs inside the agent-worker API server, relaying
+ // completions on behalf of many remote agent workers. Such a processor must
+ // never kill itself: dropping its receiver would disconnect the shared
+ // job-completed channel and make every future /send_result fail until the
+ // whole server is restarted.
+ is_agent_server: bool,
stats_map: JobStatsMap,
) -> JoinHandle<()> {
tokio::spawn(async move {
@@ -376,6 +382,7 @@ pub fn start_background_processor(
jc.job.kind,
JobKind::Dependencies | JobKind::FlowDependencies
);
+ let jc_id = jc.job.id;
#[cfg(feature = "benchmark")]
let bench_job_id = jc.job.id;
#[cfg(feature = "benchmark")]
@@ -403,9 +410,20 @@ pub fn start_background_processor(
.await;
if is_init_script && !final_success {
- tracing::error!("init script errored, exiting");
- killpill_tx.send();
- break;
+ if is_agent_server {
+ // The failed init script belongs to a remote agent
+ // worker, not to this server. That worker handles its
+ // own restart; killing the server relay here would
+ // strand every other agent worker's completions.
+ tracing::error!(
+ job_id = %jc_id,
+ "agent worker init script errored; failure recorded, keeping server bg processor alive"
+ );
+ } else {
+ tracing::error!("init script errored, exiting");
+ killpill_tx.send();
+ break;
+ }
}
if is_dependency_job && is_dedicated_worker {
tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs
index 2335a78453..a4672145bb 100644
--- a/backend/windmill-worker/src/worker.rs
+++ b/backend/windmill-worker/src/worker.rs
@@ -2269,6 +2269,7 @@ pub async fn run_worker(
worker_name.clone(),
killpill_tx.clone(),
is_dedicated_worker,
+ false,
stats_map,
)),
_ => None,
diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts
index b7a8d0c397..cc27b34abc 100644
--- a/benchmarks/lib.ts
+++ b/benchmarks/lib.ts
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
-export const VERSION = "v1.753.0";
+export const VERSION = "v1.757.0";
export async function login(email: string, password: string): Promise {
return await windmill.UserService.login({
diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts
index 71d90af6fe..17a5e1febb 100644
--- a/cli/src/commands/flow/flow.ts
+++ b/cli/src/commands/flow/flow.ts
@@ -405,6 +405,7 @@ async function run(
opts: GlobalOptions & {
data?: string;
silent: boolean;
+ tag?: string;
},
path: string
) {
@@ -433,6 +434,7 @@ async function run(
const id = await wmill.runFlowByPath({
workspace: workspace.workspaceId,
path,
+ tag: opts.tag,
requestBody: input,
});
@@ -587,6 +589,7 @@ async function preview(
silent: boolean;
remote?: boolean;
step?: string;
+ tag?: string;
} & SyncOptions,
flowPath: string
) {
@@ -699,7 +702,7 @@ async function preview(
const flowWmPath = stripFlowSuffix(flowPath).replaceAll(SEP, "/");
if (opts.step) {
- await previewStep(opts.step, localFlow, flowWmPath, workspace, input, tempScriptRefs, opts.silent);
+ await previewStep(opts.step, localFlow, flowWmPath, workspace, input, tempScriptRefs, opts.silent, opts.tag);
return;
}
@@ -714,6 +717,7 @@ async function preview(
value: localFlow.value,
path: flowWmPath,
args: input,
+ tag: opts.tag,
temp_script_refs: tempScriptRefs,
},
});
@@ -747,6 +751,7 @@ async function previewStep(
baseArgs: Record,
tempScriptRefs: Record | undefined,
silent: boolean,
+ tag: string | undefined,
) {
const module = findStepInFlowValue(localFlow.value, stepId);
if (!module) {
@@ -778,6 +783,7 @@ async function previewStep(
path: `${flowWmPath}/${stepId}`,
flow_path: flowWmPath,
args,
+ tag,
temp_script_refs: tempScriptRefs,
},
});
@@ -804,6 +810,7 @@ async function previewStep(
path: moduleValue.path,
flow_path: flowWmPath,
args,
+ tag,
temp_script_refs: tempScriptRefs,
},
});
@@ -812,6 +819,7 @@ async function previewStep(
jobId = await wmill.runFlowByPath({
workspace: workspace.workspaceId,
path: moduleValue.path,
+ tag,
requestBody: args,
});
} else {
@@ -1122,6 +1130,10 @@ const command = new Command()
"-s --silent",
"Do not ouput anything other then the final output. Useful for scripting."
)
+ .option(
+ "--tag ",
+ "Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the flow's default tag)."
+ )
.action(run as any)
.command(
"preview",
@@ -1144,6 +1156,10 @@ const command = new Command()
"--step ",
"Run only the named step instead of the whole flow. Honors --data as the step's args and --remote / local-PathScript resolution the same way the full-flow preview does."
)
+ .option(
+ "--tag ",
+ "Override the worker tag the preview is dispatched to (e.g. to route it to dev workers instead of the flow's default tag)."
+ )
.action(preview as any)
.command(
"generate-locks",
diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts
index ed57b1747d..3db9c7b53e 100644
--- a/cli/src/commands/script/script.ts
+++ b/cli/src/commands/script/script.ts
@@ -1045,6 +1045,7 @@ async function run(
opts: GlobalOptions & {
data?: string;
silent: boolean;
+ tag?: string;
},
path: string
) {
@@ -1075,6 +1076,7 @@ async function run(
id = await wmill.runScriptByPath({
workspace: workspace.workspaceId,
path,
+ tag: opts.tag,
requestBody: input,
});
} catch (e: any) {
@@ -1486,6 +1488,7 @@ async function preview(
opts: GlobalOptions & {
data?: string;
silent: boolean;
+ tag?: string;
} & SyncOptions,
filePath: string
) {
@@ -1647,6 +1650,7 @@ async function preview(
path: filePath.substring(0, filePath.indexOf(".")).replaceAll(SEP, "/"),
args: input,
language: language,
+ tag: opts.tag,
kind: isTar ? "tarbundle" : "bundle",
format: codebase?.format ?? "cjs",
temp_script_refs: tempScriptRefs,
@@ -1716,6 +1720,7 @@ async function preview(
path: filePath.substring(0, filePath.indexOf(".")).replaceAll(SEP, "/"),
args: input,
language: language as any,
+ tag: opts.tag,
modules: modules ?? undefined,
temp_script_refs: tempScriptRefs,
},
@@ -1842,6 +1847,10 @@ const command = new Command()
"-s --silent",
"Do not output anything other then the final output. Useful for scripting."
)
+ .option(
+ "--tag ",
+ "Override the worker tag the run is dispatched to (e.g. to route it to dev workers instead of the script's default tag)."
+ )
.action(run as any)
.command(
"preview",
@@ -1856,6 +1865,10 @@ const command = new Command()
"-s --silent",
"Do not output anything other than the final output. Useful for scripting."
)
+ .option(
+ "--tag