starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-08 00:03:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into change-fd3c58d1

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
#	backend/windmill-common/src/workspaces.rs
#	backend/windmill-trigger-postgres/src/lib.rs
This commit is contained in:
Diego Imbert
2026-07-24 15:58:18 +02:00
parent 785dd0fdf6 a26ea4d43f
commit ff8e4c4836
296 changed files with 25962 additions and 2996 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.agents/skills/pr/SKILL.md
+3 -1
View File
@@ -139,8 +139,10 @@ A PR leaves draft **only after a clean CI review round**. Never run `gh pr ready
It comments `/review` on the PR — which runs the Codex, Claude and Pi CI reviewers even on a draft — waits for the spawned `PR Review Commands` workflow run(s) to complete, then prints one verdict line per reviewer and saves the full review comments to files.
`/review` (and `/codex`) are **idempotent per head SHA**: if a running or successful review already covers the current head, they skip that agent and post nothing new — the waiter reads the existing verdict for that head, so a skipped agent is *not* a missing one. A cancelled/failed head run is re-run in place; a fresh run is launched only when nothing covers the head. So an unchanged-head re-review is a near no-op, not a new round — push a commit to get genuinely fresh reviews.
2. **Judge the round.** Codex is mandatory; Claude, Pi and cubic count whenever they posted. Every review starts with one of the three `REVIEW.md` verdicts:
- Codex verdict missing → the round is void: comment `/codex` on the PR, wait for it the same way, and judge again.
- Codex verdict missing → the round is void: the waiter warns only when the head has no green Codex run (cancelled/failed/absent — not merely skipped-because-already-reviewed). Comment `/codex` on the PR, which re-runs the interrupted run in place (or launches one if none exists), wait the same way, and judge again.
- Any **"Should address issues before merging"** → fix the P0/P1 findings (and the nits while you're there), commit, push, and start a new round (step 1).
- Only **"Mergeable, but should ideally address nits"** and/or **"Good to merge"** → fix the nits too; a nit that is wrong or genuinely not worth fixing may instead be dismissed by replying to the review comment with your reasoning. Push nit-only fixes without starting another full round.
.agents/skills/pr/review-round.sh
+55 -12
View File
@@ -79,10 +79,43 @@ while :; do
sleep 60
done
# Head SHA at trigger time. `/review` is idempotent per head: it skips an agent a
# running/successful review already covers, re-runs a cancelled/failed one in place on a
# separate head-tied run, and launches fresh only when nothing covers the head. Verdict
# reading below therefore keys off the head, not just the trigger timestamp.
HEAD_SHA=$(retry gh api "repos/$REPO/pulls/$PR" --jq .head.sha)
echo "Reviewing head $HEAD_SHA"
# Newest non-skipped run of <workflow> tied to the head ("status conclusion"), or empty
# when none exists. A re-run-in-place or an already-covering review resolves on such a
# head-tied run — separate from the pr-review-commands run waited on above (a fresh
# launch instead runs inside it, and posts after the trigger). A `skipped` run is the
# draft/fork gate and produced no review, so it is ignored.
head_run_state() {
gh run list --repo "$REPO" --workflow "$1" --commit "$HEAD_SHA" --limit 20 \
--json databaseId,status,conclusion \
--jq '[.[] | select(.conclusion != "skipped")] | sort_by(.databaseId) | last | if . then "\(.status) \(.conclusion // "-")" else empty end' 2>/dev/null || true
}
# A re-run-in-place review lands on a head-tied run that finishes after the fast
# pr-review-commands run, so let those settle before reading verdicts.
for wf in codex-pr-review.yml pi-pr-review.yml pr-ready-review.yml; do
while :; do
case "$(head_run_state "$wf")" in
""|"completed "*) break ;;
*) if [ "$(date +%s)" -gt "$DEADLINE" ]; then break; fi; sleep 30 ;;
esac
done
done
OUT_DIR=$(mktemp -d -t review-round-XXXXXX)
COMMENTS_RAW=$(retry gh api "repos/$REPO/issues/$PR/comments?per_page=100" --paginate)
# Two views: comments from THIS round (after the trigger) and the full history. A fresh
# launch posts after the trigger; an idempotent skip leaves the covering verdict in the
# earlier run's comment, so fall back to history when that agent's head run is green.
jq -s --arg t "$TRIGGER_TIME" '[.[][] | select(.created_at > $t)]' \
<<<"$COMMENTS_RAW" > "$OUT_DIR/comments.json"
jq -s '[.[][]]' <<<"$COMMENTS_RAW" > "$OUT_DIR/comments-all.json"
# cubic posts through the PR reviews API, not issue comments.
REVIEWS_RAW=$(retry gh api "repos/$REPO/pulls/$PR/reviews?per_page=100" --paginate)
jq -s --arg t "$TRIGGER_TIME" '[.[][] | select((.submitted_at // "") > $t)]' \
@@ -90,18 +123,28 @@ jq -s --arg t "$TRIGGER_TIME" '[.[][] | select((.submitted_at // "") > $t)]' \
VERDICT_RE='(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)'
body_by_header() {
jq -r --arg h "$1" '[.[] | select(.body // "" | contains($h))] | last | .body // empty' \
"$OUT_DIR/comments.json"
body_by_header() { # <file> <header-substring>
jq -r --arg h "$2" '[.[] | select(.body // "" | contains($h))] | last | .body // empty' "$1"
}
body_by_login() {
jq -r --arg l "$1" '[.[] | select(.user.login == $l)] | last | .body // empty' \
"$OUT_DIR/comments.json"
body_by_login() { # <file> <login>
jq -r --arg l "$2" '[.[] | select(.user.login == $l)] | last | .body // empty' "$1"
}
head_ok() { [ "$(head_run_state "$1")" = "completed success" ]; }
# Latest verdict for a reviewer: prefer this round's comment; if none and the reviewer's
# head run succeeded (an idempotent /review skipped re-reviewing an already-green head),
# fall back to the covering comment from the full history.
verdict_body() { # <header|login> <value> <workflow>
local body
body=$("body_by_$1" "$OUT_DIR/comments.json" "$2")
if [ -z "$body" ] && head_ok "$3"; then
body=$("body_by_$1" "$OUT_DIR/comments-all.json" "$2")
fi
printf '%s' "$body"
}
report() { # <reviewer-name> <comment-body>
local name=$1 body=$2 verdict
if [ -z "$body" ]; then
echo "$name: (no review posted this round)"
echo "$name: (no review posted for this head)"
return
fi
printf '%s\n' "$body" > "$OUT_DIR/$name.md"
@@ -110,11 +153,11 @@ report() { # <reviewer-name> <comment-body>
}
echo
echo "=== Review round verdicts for $REPO#$PR (posted after $TRIGGER_TIME) ==="
CODEX_BODY=$(body_by_header '## Codex Review')
echo "=== Review round verdicts for $REPO#$PR (head $HEAD_SHA) ==="
CODEX_BODY=$(verdict_body header '## Codex Review' codex-pr-review.yml)
report codex "$CODEX_BODY"
report claude "$(body_by_login 'claude[bot]')"
report pi "$(body_by_header '## Pi Review')"
report claude "$(verdict_body login 'claude[bot]' pr-ready-review.yml)"
report pi "$(verdict_body header '## Pi Review' pi-pr-review.yml)"
CUBIC_BODY=$(jq -r '[.[] | select(.user.login | test("^cubic(-dev-ai)?(\\[bot\\])?$"; "i"))] | last | .body // empty' \
"$OUT_DIR/pr-reviews.json")
if [ -z "$CUBIC_BODY" ]; then
@@ -125,5 +168,5 @@ report cubic "$CUBIC_BODY"
echo
echo "Full round output: $OUT_DIR (comments.json, pr-reviews.json, one .md per reviewer)"
if [ -z "$CODEX_BODY" ]; then
echo "WARNING: Codex verdict missing - the round is incomplete. Re-trigger with a '/codex' PR comment and wait again." >&2
echo "WARNING: no Codex verdict for $HEAD_SHA - its head run is not green (cancelled/failed/absent, not merely skipped-because-already-reviewed). Re-trigger with a '/codex' PR comment (re-runs the interrupted run in place, or launches one) and wait again." >&2
fi
.github/workflows/backend-test-windows.yml
+13 -2
View File
@@ -51,6 +51,11 @@ jobs:
with:
cache-workspaces: backend
toolchain: 1.97.0
# This action defaults RUSTFLAGS to "-D warnings"; unset it so the test
# run is not failed by cross-platform dead-code (cfg(unix)-only helpers
# are unused on Windows). Warning hygiene is enforced on the Linux CI
# and the build_windows_worker_ release build, not this test job.
rustflags: ""
- uses: actions/setup-dotnet@v4
with:
@@ -203,9 +208,15 @@ jobs:
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1
# Windows ships a worker-only binary, so test the crates a worker runs
# (windmill-worker/-common/-queue) via -p, not `--all`: this skips the
# disk-heavy windmill-api test binaries (LNK1180) and the server-only
# windmill-trigger-* crates (amqp does not build on Windows). Linux CI runs the rest.
run: >
cargo test
--no-fail-fast
--features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline
--all
-p windmill-worker
-p windmill-common
-p windmill-queue
--features private,enterprise,deno_core,duckdb,python,rust,csharp,php,quickjs,parquet,mcp,scoped_cache,windmill-git-sync/private,windmill-object-store/private,windmill-object-store/enterprise
-- --nocapture --test-threads=10
.github/workflows/build_windows_worker_.yml
+5 -1
View File
@@ -45,8 +45,12 @@ jobs:
env:
RUSTFLAGS: "-D warnings"
run: |
mkdir frontend/build && cd backend
cd backend
# Stub the openapi specs to empty: they are compiled in via an ungated
# include_str! but a worker binary never serves them, so this avoids
# embedding ~2.5MB of spec.
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force
cargo check --features=ee_windows
- name: Cargo build dynamic libraries windows
.github/workflows/git-sync-test.yml
+10 -2
View File
@@ -9,6 +9,10 @@ on:
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "backend/windmill-common/src/workspaces.rs"
- "backend/windmill-worker/src/result_processor.rs"
- "backend/windmill-api-workspaces/**"
- "cli/src/commands/sync/**"
- "cli/src/utils/git.ts"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
pull_request:
@@ -18,6 +22,10 @@ on:
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "backend/windmill-common/src/workspaces.rs"
- "backend/windmill-worker/src/result_processor.rs"
- "backend/windmill-api-workspaces/**"
- "cli/src/commands/sync/**"
- "cli/src/utils/git.ts"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
@@ -50,8 +58,8 @@ jobs:
echo "Changed files:"
echo "$CHANGED_FILES"
# Direct git sync file changes — always relevant
if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
# Direct git sync file changes — always relevant.
if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Relevant: direct git sync file changes"
exit 0
.github/workflows/pr-review-commands.yml
+176 -6
View File
@@ -75,14 +75,153 @@ jobs:
"/repos/$REPO/issues/comments/$COMMENT_ID/reactions" \
-f content=eyes >/dev/null
claude:
# Decide, per agent, whether to launch a fresh run, re-run in place, or skip. A push
# already auto-triggers codex/pi (and claude on open) against the PR head. Relaunching
# via this issue_comment path both cancels those in-flight auto runs (shared concurrency
# group) AND lands the new run's status on main — issue_comment runs never attach a
# check to the PR head — leaving the PR showing only a cancelled review. So for every
# command, launch an agent only when nothing covers the head commit; if the head's run
# was cancelled/failed, re-run it in place (a re-run keeps the original pull_request
# event, so its checks re-attach to the PR head); skip when a running or successful run
# already covers it. `/review` applies this to all three agents; `/codex`, `/pi`,
# `/claude` apply the same decision to just their own agent.
plan:
needs: [parse, check-access]
if: |
needs.parse.outputs.command != '' &&
(
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) ||
needs.check-access.outputs.authorized == 'true'
)
runs-on: ubuntu-latest
permissions:
contents: read
actions: write
pull-requests: read
statuses: write
outputs:
head_sha: ${{ steps.plan.outputs.head_sha }}
launch_codex: ${{ steps.plan.outputs.launch_codex }}
launch_pi: ${{ steps.plan.outputs.launch_pi }}
launch_claude: ${{ steps.plan.outputs.launch_claude }}
steps:
- name: Decide per-agent launch vs re-run for the head commit
id: plan
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.issue.number }}
COMMAND: ${{ needs.parse.outputs.command }}
run: |
set -euo pipefail
HEAD_SHA=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid')
echo "PR #$PR_NUMBER head: $HEAD_SHA"
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
RUN_URL="$GITHUB_SERVER_URL/$REPO/actions/runs/$GITHUB_RUN_ID"
# A fresh launch runs from this issue_comment workflow (associated with main),
# so it never appears in the PR-head run query below and its own check lands on
# main, not the head. To keep fresh launches idempotent per head, mark the head
# SHA with a `review-launch/<agent>` commit status at launch; the `finalize` job
# resolves it to success/failure. A prior launch's status covering the head lets
# a second comment skip instead of relaunching (which would cancel the first via
# the reviewer's shared concurrency group). All status calls are best-effort — a
# GitHub API hiccup must degrade to a relaunch, never abort the decision.
mark_launch() {
agent="$1"
gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \
-f state=pending -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \
-f "description=Review launched via /$COMMAND" >/dev/null 2>&1 || true
}
# Returns "covered" if a prior fresh launch (this or an earlier comment run)
# already covers the head: a success status, or a pending status whose launching
# run is still alive. A pending whose run has completed is stale (that run
# crashed before finalize) and does not count.
launch_coverage() {
agent="$1"
st_json=$(gh api "repos/$REPO/commits/$HEAD_SHA/statuses" \
--jq "[.[] | select(.context == \"review-launch/$agent\")] | first // empty" 2>/dev/null || true)
[ -n "$st_json" ] || return 0
state=$(jq -r '.state // empty' <<<"$st_json" 2>/dev/null || true)
[ "$state" = success ] && { echo covered; return 0; }
[ "$state" = pending ] || return 0
target=$(jq -r '.target_url // empty' <<<"$st_json" 2>/dev/null || true)
run_id=$(printf '%s' "$target" | grep -oE '[0-9]+$' || true)
if [ -n "$run_id" ]; then
run_state=$(gh run view "$run_id" --repo "$REPO" --json status --jq '.status' 2>/dev/null || true)
[ "$run_state" = completed ] && return 0 # stale pending -> not covered
fi
echo covered
}
decide() {
wf="$1"; key="$2"; agent="$3"
if [ "$(launch_coverage "$agent")" = covered ]; then
echo "$key: a prior launch already covers $HEAD_SHA (review-launch/$agent) -> skip"
echo "$key=false" >> "$GITHUB_OUTPUT"
return
fi
# `--commit` matches runs whose head SHA is the PR head. Auto reviews run on
# `pull_request` against that SHA; `/review` (issue_comment) runs execute on
# main, so they never match and are not counted as covering the head commit.
runs=$(gh run list --repo "$REPO" --workflow "$wf" --commit "$HEAD_SHA" --limit 40 \
--json databaseId,status,conclusion)
# Healthy = still running, or completed successfully: a review already
# covers this commit, so skip.
healthy=$(jq -r '[.[] | select(.status != "completed" or .conclusion == "success")] | length' <<<"$runs")
if [ "$healthy" -gt 0 ]; then
echo "$key: a running or successful review already covers $HEAD_SHA -> skip"
echo "$key=false" >> "$GITHUB_OUTPUT"
return
fi
# Re-run only genuinely interrupted runs (cancelled/failed/timed out) in
# place, so their checks re-attach to the PR head instead of posting on
# main. A `skipped` run produced no review and would just skip again (it is
# the draft/fork gate), so it does not count — fall through to a fresh launch.
retry_id=$(jq -r '[.[] | select(.status == "completed" and (.conclusion == "cancelled" or .conclusion == "failure" or .conclusion == "timed_out"))] | sort_by(.databaseId) | last | .databaseId // empty' <<<"$runs")
if [ -n "$retry_id" ]; then
if gh run rerun "$retry_id" --repo "$REPO" >/dev/null 2>&1; then
echo "$key: re-ran interrupted run $retry_id (re-attaches to PR head)"
echo "$key=false" >> "$GITHUB_OUTPUT"
return
fi
echo "$key: re-run of $retry_id failed -> fresh launch"
mark_launch "$agent"
echo "$key=true" >> "$GITHUB_OUTPUT"
return
fi
echo "$key: no usable review for $HEAD_SHA -> launch"
mark_launch "$agent"
echo "$key=true" >> "$GITHUB_OUTPUT"
}
# `/review` targets all three agents; `/codex`, `/pi`, `/claude` target only
# their own. A non-targeted agent is left untouched (no launch, no re-run).
decide_if_targeted() {
wf="$1"; key="$2"; agent="$3"
if [ "$COMMAND" = review ] || [ "$COMMAND" = "$agent" ]; then
decide "$wf" "$key" "$agent"
else
echo "$key: /$COMMAND does not target $agent -> skip"
echo "$key=false" >> "$GITHUB_OUTPUT"
fi
}
decide_if_targeted codex-pr-review.yml launch_codex codex
decide_if_targeted pi-pr-review.yml launch_pi pi
decide_if_targeted pr-ready-review.yml launch_claude claude
claude:
needs: [parse, check-access, plan]
if: |
(
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) ||
needs.check-access.outputs.authorized == 'true'
) &&
(needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'claude')
needs.plan.outputs.launch_claude == 'true'
permissions:
contents: read
pull-requests: read
@@ -97,13 +236,13 @@ jobs:
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
codex:
needs: [parse, check-access]
needs: [parse, check-access, plan]
if: |
(
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) ||
needs.check-access.outputs.authorized == 'true'
) &&
(needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'codex')
needs.plan.outputs.launch_codex == 'true'
permissions:
contents: read
issues: write
@@ -119,13 +258,13 @@ jobs:
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
pi:
needs: [parse, check-access]
needs: [parse, check-access, plan]
if: |
(
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) ||
needs.check-access.outputs.authorized == 'true'
) &&
(needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'pi')
needs.plan.outputs.launch_pi == 'true'
permissions:
contents: read
issues: write
@@ -138,3 +277,34 @@ jobs:
secrets:
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
# Resolve the `review-launch/<agent>` head statuses that `plan` set to pending, so a
# fresh launch's outcome is visible on the PR head (not just on main) and never lingers
# as a stale pending check. Targets the exact SHA `plan` launched against, so a push
# that moved the head mid-review does not stamp a status on the new head.
finalize:
needs: [plan, claude, codex, pi]
if: always() && needs.plan.result == 'success' && needs.plan.outputs.head_sha != ''
runs-on: ubuntu-latest
permissions:
statuses: write
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
HEAD_SHA: ${{ needs.plan.outputs.head_sha }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- name: Finalize launch statuses on the PR head
run: |
set -uo pipefail
finalize() {
agent="$1"; launched="$2"; result="$3"
[ "$launched" = true ] || return 0
state=$([ "$result" = success ] && echo success || echo failure)
gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \
-f "state=$state" -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \
-f "description=Review $result" >/dev/null 2>&1 || true
}
finalize codex "${{ needs.plan.outputs.launch_codex }}" "${{ needs.codex.result }}"
finalize pi "${{ needs.plan.outputs.launch_pi }}" "${{ needs.pi.result }}"
finalize claude "${{ needs.plan.outputs.launch_claude }}" "${{ needs.claude.result }}"
.github/workflows/publish_windows_worker.yml
+5 -1
View File
@@ -56,8 +56,12 @@ jobs:
vcpkg.exe integrate install
$env:VCPKGRS_DYNAMIC=1
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
cd backend
# Stub the openapi specs to empty: they are compiled in via an ungated
# include_str! but a worker binary never serves them, so this avoids
# embedding ~2.5MB of spec.
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force
cargo build --release --features=ee_windows
- name: Rename binary with corresponding architecture
run: |
CHANGELOG.md
+123
View File
@@ -1,5 +1,128 @@
# Changelog
## [1.769.0](https://github.com/windmill-labs/windmill/compare/v1.768.0...v1.769.0) (2026-07-24)
### Features
* **hub:** surface data pipelines in deploy-to-hub drawer ([#10299](https://github.com/windmill-labs/windmill/issues/10299)) ([010059a](https://github.com/windmill-labs/windmill/commit/010059a449f4d0cde0656086f1b3f03a02a1c78c))
* **pipeline:** collapse secondary top-bar controls into an overflow menu ([#10300](https://github.com/windmill-labs/windmill/issues/10300)) ([1d25d75](https://github.com/windmill-labs/windmill/commit/1d25d7539ed7fb17c47a732d6b1cd7fc19202a04))
### Bug Fixes
* pin table actions column so it stays visible on narrow screens ([#10301](https://github.com/windmill-labs/windmill/issues/10301)) ([75acf72](https://github.com/windmill-labs/windmill/commit/75acf7207b151f0227600be555bfa5e14dfa8dd1))
* pin validated DNS address to close SSRF DNS-rebinding TOCTOU ([#10303](https://github.com/windmill-labs/windmill/issues/10303)) ([3cf7a39](https://github.com/windmill-labs/windmill/commit/3cf7a390a37c07a818248c95defa4ecf5bf262e5))
### Performance Improvements
* optimize get_datatable_full_schema to avoid timeout on large catalogs ([#10304](https://github.com/windmill-labs/windmill/issues/10304)) ([1478d12](https://github.com/windmill-labs/windmill/commit/1478d12eb352b1b7906ccfe3ab5eaafe60ffbe4f))
## [1.768.0](https://github.com/windmill-labs/windmill/compare/v1.767.0...v1.768.0) (2026-07-24)
### Features
* Add section to deploy projects to hub ([#9332](https://github.com/windmill-labs/windmill/issues/9332)) ([30eedf9](https://github.com/windmill-labs/windmill/commit/30eedf9ee1754cb2bbf46e82d2b3eac766bd3e6d))
* **ai-chat:** let the session/global chat create email triggers ([#10282](https://github.com/windmill-labs/windmill/issues/10282)) ([248c875](https://github.com/windmill-labs/windmill/commit/248c8751b1fd5d6f7506a7c3fc8f500dd98af26e))
* **ai:** enable data pipelines in AI sessions with alpha notice ([#10273](https://github.com/windmill-labs/windmill/issues/10273)) ([1f912f4](https://github.com/windmill-labs/windmill/commit/1f912f410401ec68789d43973d6e0a1f8e595f86))
* alert on expired online license key ([#10295](https://github.com/windmill-labs/windmill/issues/10295)) ([30d8104](https://github.com/windmill-labs/windmill/commit/30d8104edcbdb7f0c63f186d22ac03000e0d4f07))
* data-pipeline recorder, interactive player, and deploy-to-hub recording (WIN-2156) ([#10055](https://github.com/windmill-labs/windmill/issues/10055)) ([65e5041](https://github.com/windmill-labs/windmill/commit/65e504146d832b105603ceec89c6de0ad1d66e32))
* let a workspace fall back to the instance critical alert channels ([#10292](https://github.com/windmill-labs/windmill/issues/10292)) ([717e38a](https://github.com/windmill-labs/windmill/commit/717e38a0c6b5bb2340e236a9d49645a4cebf4849))
* **monitor:** make between-steps zombie flows hand-recoverable ([#10287](https://github.com/windmill-labs/windmill/issues/10287)) ([f02df7f](https://github.com/windmill-labs/windmill/commit/f02df7fc454b0c2a2afa9ae9848e26af0e379246))
* surface workspace-script advanced settings in flow editor ([#10289](https://github.com/windmill-labs/windmill/issues/10289)) ([bf16e7d](https://github.com/windmill-labs/windmill/commit/bf16e7d49a7486d37cf9eb1907e80abae47a78a7))
* **windows:** enable ruby and rlang on the windows worker ([#10279](https://github.com/windmill-labs/windmill/issues/10279)) ([0f1b864](https://github.com/windmill-labs/windmill/commit/0f1b8641f26f0eda8a8517db1a37efafdb173464))
### Bug Fixes
* **ci:** make /review idempotent per head commit, re-run cancelled reviews in place ([#10283](https://github.com/windmill-labs/windmill/issues/10283)) ([3aaceb7](https://github.com/windmill-labs/windmill/commit/3aaceb7efb3bd232da73b79fcdcf4c712caad7d3))
* **cli:** surface shared UI changes in sync push dry-run preview ([#10278](https://github.com/windmill-labs/windmill/issues/10278)) ([14c29b7](https://github.com/windmill-labs/windmill/commit/14c29b77e90842b8c3d4395ce50cb44666cd998c))
* **jobs:** sanitize NUL in completed job result before jsonb insert ([#10274](https://github.com/windmill-labs/windmill/issues/10274)) ([c50a2ab](https://github.com/windmill-labs/windmill/commit/c50a2abad0c222bf8f76ff10b864450120ee9ee7))
* **monitor:** diagnose zombie-flow OOM on the transition worker, not q.worker ([#10286](https://github.com/windmill-labs/windmill/issues/10286)) ([fa36442](https://github.com/windmill-labs/windmill/commit/fa3644281f30d90aa0c4ca86520a38db5d8b91f0))
* resolve svelte/style export conditions in raw-app CLI bundler ([#10294](https://github.com/windmill-labs/windmill/issues/10294)) ([9713e60](https://github.com/windmill-labs/windmill/commit/9713e6074d2df55db1331cf31ee1357606b1dbe9))
* **resources:** apply resource_type changes on update (git-sync pull) — Fixes GIT-932 ([#10277](https://github.com/windmill-labs/windmill/issues/10277)) ([d7a0078](https://github.com/windmill-labs/windmill/commit/d7a0078b58f74f1ffecc03b348e8935a70f02e55))
* surface workspace ids on duplicate names and explain fork promotion ([#10291](https://github.com/windmill-labs/windmill/issues/10291)) ([9b182aa](https://github.com/windmill-labs/windmill/commit/9b182aaf3879d4d81b6a785222dca75f085e6fb7))
* treat concurrent_limit/timeout &lt;= 0 as unset instead of a zero cap ([#10288](https://github.com/windmill-labs/windmill/issues/10288)) ([8eb36ce](https://github.com/windmill-labs/windmill/commit/8eb36ce008b4efe2be9a9bfebc91af68070f7a6c))
## [1.767.0](https://github.com/windmill-labs/windmill/compare/v1.766.2...v1.767.0) (2026-07-22)
### Features
* **ai-chat:** email triggers in flow/script chat + trigger-intent eval guards (WIN-2228) ([#10267](https://github.com/windmill-labs/windmill/issues/10267)) ([e41440b](https://github.com/windmill-labs/windmill/commit/e41440b344d94ebe5746a89237c3d1be522aa59d))
* **ai:** improve data-pipeline building in AI sessions (prompt + evals + e2e) ([#10270](https://github.com/windmill-labs/windmill/issues/10270)) ([0819641](https://github.com/windmill-labs/windmill/commit/0819641f3a89abdf114cc5e341a9f585e5d71296))
### Bug Fixes
* **ai-chat:** improve resource-type search tool description and scoring ([#10272](https://github.com/windmill-labs/windmill/issues/10272)) ([ad53673](https://github.com/windmill-labs/windmill/commit/ad53673a2855e787af9fe10dcc8998ffc7f1b960))
* **embeddings:** retry on failed init instead of disabling for a day ([#10266](https://github.com/windmill-labs/windmill/issues/10266)) ([2318481](https://github.com/windmill-labs/windmill/commit/2318481f4f2932dc578d5cb64c4c705cf5c324a0))
* manual resource type sync fetches from hub first, cache as fallback ([#10269](https://github.com/windmill-labs/windmill/issues/10269)) ([07d4b67](https://github.com/windmill-labs/windmill/commit/07d4b674f1dbd24c89f7d1e77094a40e528a3740))
## [1.766.2](https://github.com/windmill-labs/windmill/compare/v1.766.1...v1.766.2) (2026-07-22)
### Bug Fixes
* **python,windows:** cross-platform cross-process wheel-install lock ([#10264](https://github.com/windmill-labs/windmill/issues/10264)) ([7e2f1af](https://github.com/windmill-labs/windmill/commit/7e2f1afffb4982099870e7ce32592cf3a447e513))
## [1.766.1](https://github.com/windmill-labs/windmill/compare/v1.766.0...v1.766.1) (2026-07-22)
### Bug Fixes
* **jobs:** enforce self_approval_disabled on the UI resume path ([#10262](https://github.com/windmill-labs/windmill/issues/10262)) ([2d24b3a](https://github.com/windmill-labs/windmill/commit/2d24b3ac49b65469072bca929730d2c9ecd54b8f))
## [1.766.0](https://github.com/windmill-labs/windmill/compare/v1.765.0...v1.766.0) (2026-07-22)
### Features
* **ai-sessions:** show item preview cards for tools ([#10254](https://github.com/windmill-labs/windmill/issues/10254)) ([703744f](https://github.com/windmill-labs/windmill/commit/703744fb8b76f70384cbc3341d5e1b3dbf30455e))
* make content search a full CE feature ([#10252](https://github.com/windmill-labs/windmill/issues/10252)) ([d2c5d6f](https://github.com/windmill-labs/windmill/commit/d2c5d6f4b4ede9449407ed3831713d3e3d1d6972))
* **sessions:** ship AI sessions as beta with legacy-chat opt-out ([#10242](https://github.com/windmill-labs/windmill/issues/10242)) ([0508cdd](https://github.com/windmill-labs/windmill/commit/0508cddf0a86b6a6a043cb31978ca68b76a95d02))
### Bug Fixes
* accept ssh/scheme-less git repo urls and $var: refs in app repo resolution ([#10246](https://github.com/windmill-labs/windmill/issues/10246)) ([b948efd](https://github.com/windmill-labs/windmill/commit/b948efd3c81aada103bd141eae03fbe626bf6ee5))
* **copilot:** stop write_flow forcing rawscript code into nested JSON ([#10260](https://github.com/windmill-labs/windmill/issues/10260)) ([ecb1a92](https://github.com/windmill-labs/windmill/commit/ecb1a92070fb18048a6a1467270469e8462a3cbf))
* prevent u16 underflow in suspend count causing permanent flow deadlock ([#10256](https://github.com/windmill-labs/windmill/issues/10256)) ([68b1fcc](https://github.com/windmill-labs/windmill/commit/68b1fcc5cdd3d217ebb1b9e09c0d52334faf0a63))
* **prompts:** prefer Bun over Deno for TypeScript runtime selection ([#10253](https://github.com/windmill-labs/windmill/issues/10253)) ([380cf75](https://github.com/windmill-labs/windmill/commit/380cf752ca8fc57eb70f7c8efd57290f7902a030))
* **tutorials:** repair broken frontend tutorials after UI redesigns ([#10255](https://github.com/windmill-labs/windmill/issues/10255)) ([5685981](https://github.com/windmill-labs/windmill/commit/5685981c9902ff37c3bd271dc5c5ede948a0787a))
## [1.765.0](https://github.com/windmill-labs/windmill/compare/v1.764.0...v1.765.0) (2026-07-21)
### Features
* **ai:** open the Compare & Deploy page from chat with item preselection ([#10232](https://github.com/windmill-labs/windmill/issues/10232)) ([572d69e](https://github.com/windmill-labs/windmill/commit/572d69e5ae8ae12207dca175fdb5ea378d852a45))
* **ai:** session chat nits — empty sends, command picker polish, session-state prompt ([#10233](https://github.com/windmill-labs/windmill/issues/10233)) ([9bc1f62](https://github.com/windmill-labs/windmill/commit/9bc1f6212837155c964a7db7103aabbcf14cbce2))
* attach text files to chat messages, read on demand via file tools ([#10215](https://github.com/windmill-labs/windmill/issues/10215)) ([d6cf1ef](https://github.com/windmill-labs/windmill/commit/d6cf1ef9872cd0db0f51e4881237bac383e45f61))
* **git-sync:** enable per-item promotion mode on dev workspaces ([#10205](https://github.com/windmill-labs/windmill/issues/10205)) ([2ce21c9](https://github.com/windmill-labs/windmill/commit/2ce21c9ef86c608d73e12a7e278d2f328e06b6fa))
* **pipelines:** catalog declared measures and dimensions ([#10190](https://github.com/windmill-labs/windmill/issues/10190)) ([fd51d40](https://github.com/windmill-labs/windmill/commit/fd51d40f1254a206de1f38e59cc3dadce8e13aa5))
* **triggers:** add AMQP (RabbitMQ) trigger via lapin ([#10230](https://github.com/windmill-labs/windmill/issues/10230)) ([68debab](https://github.com/windmill-labs/windmill/commit/68debab877c6dc8ee3732e0c23d467db85fd4584))
* unified read-only `diff` chat tool (drafts, fork vs parent, search) ([#10211](https://github.com/windmill-labs/windmill/issues/10211)) ([9739d5a](https://github.com/windmill-labs/windmill/commit/9739d5a2c2a21beded8848ea9049b2816196a898))
### Bug Fixes
* **apps:** let entitled viewers read pre-existing S3 files from deployed apps ([#10245](https://github.com/windmill-labs/windmill/issues/10245)) ([4a89824](https://github.com/windmill-labs/windmill/commit/4a898247a21ae918fa952a993fbe407ebc49e404))
* **db:** repair s3 asset paths missing default-storage leading slash ([#10243](https://github.com/windmill-labs/windmill/issues/10243)) ([555c751](https://github.com/windmill-labs/windmill/commit/555c751016fea4087be09fa2520af331cc872bfc))
* **frontend:** curl fallback for +Variable/+Resource in bash sandbox mode ([#10235](https://github.com/windmill-labs/windmill/issues/10235)) ([28966bd](https://github.com/windmill-labs/windmill/commit/28966bdbf190ac461aa5259e04d6c15cf699fd94))
* **frontend:** limit compare & deploy rows to the active direction ([#10234](https://github.com/windmill-labs/windmill/issues/10234)) ([7ac27c1](https://github.com/windmill-labs/windmill/commit/7ac27c1ef240729fc38d5fa506b96bf60be74e25))
* **frontend:** prevent browser back-swipe navigation over monaco editors ([#10229](https://github.com/windmill-labs/windmill/issues/10229)) ([b0bf256](https://github.com/windmill-labs/windmill/commit/b0bf25683ba3aac3e94bbe9d6bbd6e22545e734e))
* **parsers:** keep s3 asset path suffix verbatim to preserve storage distinction ([#10241](https://github.com/windmill-labs/windmill/issues/10241)) ([7fb8a2e](https://github.com/windmill-labs/windmill/commit/7fb8a2e390cef3cd01a33c89ee55d3f53cb1e20f))
* **parser:** spurious pg arg inferred from placeholders in comments ([#10226](https://github.com/windmill-labs/windmill/issues/10226)) ([d24e176](https://github.com/windmill-labs/windmill/commit/d24e1768163c10fbbc144bd1126457e42d282e37))
* **pg:** actionable error when s3object input exceeds jsonb 256MB cap ([#10228](https://github.com/windmill-labs/windmill/issues/10228)) ([6e42633](https://github.com/windmill-labs/windmill/commit/6e4263364325e4d4616697c6da601bbc887ff43c))
* **postgres-triggers:** enforce resource-path scopes on ancillary routes ([#10222](https://github.com/windmill-labs/windmill/issues/10222)) ([39058c0](https://github.com/windmill-labs/windmill/commit/39058c0a01c35cf553daae3af37c6d326a7c8763))
* return to parent workspace when a fork is deleted remotely ([#9898](https://github.com/windmill-labs/windmill/issues/9898)) ([32994df](https://github.com/windmill-labs/windmill/commit/32994df427e6df6747caf1ba6371512d5a5be5c2))
* **s3:** support instance-policy credentials in object storage tests ([#10238](https://github.com/windmill-labs/windmill/issues/10238)) ([ec63244](https://github.com/windmill-labs/windmill/commit/ec6324409d6c0e9ddcda5d110ac570ca79fe9e38))
* **triggers:** apply scope-path filtering to list and fix update scope check ([#10220](https://github.com/windmill-labs/windmill/issues/10220)) ([adc555d](https://github.com/windmill-labs/windmill/commit/adc555d1722cb59aa5b88503b3105357d76f5b71))
* **worker:** mount /dev/shm as tmpfs in the Docker v2 nsjail sandbox ([#10240](https://github.com/windmill-labs/windmill/issues/10240)) ([2caee41](https://github.com/windmill-labs/windmill/commit/2caee41fdfe1e69010a1f4544d45aaa5db49f590))
## [1.764.0](https://github.com/windmill-labs/windmill/compare/v1.763.0...v1.764.0) (2026-07-20)
ai_evals/adapters/frontend/mockBackend.ts
+33 -12
View File
@@ -304,13 +304,20 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string {
*/
const benchmarkDrafts = new Map<
string,
{ workspace: string; kind: UserDraftItemKind; path: string; value: unknown }
{ workspace: string; kind: UserDraftItemKind; path: string; value: unknown; createdAt: string }
>()
// Fixed timestamp so artifacts stay deterministic. No eval simulates a
// concurrent writer, so every save is accepted and the conflict branch is
// never taken — the syncer just records this as its `last_sync` baseline.
const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z'
// Counter-based timestamps: deterministic run-to-run (same event order → same
// values) but MONOTONIC per update, because production bumps a draft row's
// created_at on every upsert and the diff snapshot cache keys patch reuse on
// it — a fixed timestamp would serve stale patches after an edit. No eval
// simulates a concurrent writer, so every save is accepted and the conflict
// branch is never taken.
let benchmarkDraftClock = 0
function nextBenchmarkDraftTimestamp(): string {
benchmarkDraftClock += 1
return new Date(benchmarkDraftClock * 1000).toISOString()
}
function benchmarkDraftKey(workspace: string, kind: string, path: string): string {
return `${workspace}::${kind}::${path}`
@@ -341,7 +348,8 @@ export function seedBenchmarkDraft(
workspace,
kind,
path,
value
value,
createdAt: nextBenchmarkDraftTimestamp()
})
}
@@ -354,6 +362,7 @@ export function updateBenchmarkDraft(input: {
}): UpdateDraftResponse {
const key = benchmarkDraftKey(input.workspace, input.kind, input.path)
const value = input.requestBody?.value
const createdAt = nextBenchmarkDraftTimestamp()
if (value == null) {
benchmarkDrafts.delete(key)
} else {
@@ -361,10 +370,11 @@ export function updateBenchmarkDraft(input: {
workspace: input.workspace,
kind: input.kind,
path: input.path,
value
value,
createdAt
})
}
return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP }
return { status: 'saved', current_timestamp: createdAt }
}
/** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the
@@ -378,7 +388,7 @@ export function getBenchmarkDraftForUser(input: {
if (!entry) {
throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 })
}
return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP }
return { value: entry.value, created_at: entry.createdAt }
}
/** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike
@@ -392,7 +402,18 @@ export function getBenchmarkOwnDraft(input: {
if (!entry) {
return null
}
return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP }
return { value: entry.value, created_at: entry.createdAt }
}
/** Whether a deployed benchmark item exists for a draft row's kind+path —
* drives `draft_only`, which production computes against the deployed tables. */
function benchmarkDeployedExists(workspace: string, kind: UserDraftItemKind, path: string): boolean {
if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path))
if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path))
if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path))
// Drawer kinds (variables/resources/schedules/triggers) have no deployed
// benchmark stores today.
return false
}
/** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */
@@ -403,9 +424,9 @@ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse {
kind: entry.kind,
path: entry.path,
summary: (entry.value as { summary?: string } | null)?.summary,
draft_only: true,
draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path),
legacy_draft: false,
created_at: BENCHMARK_DRAFT_TIMESTAMP
created_at: entry.createdAt
}))
}
ai_evals/adapters/frontend/vitestAdapter.test.ts
+69 -9
View File
@@ -126,13 +126,32 @@ vi.mock('$lib/gen', async () => {
hasBenchmarkWorkspace(data.workspace)
? Boolean(getBenchmarkScriptByPath(data.workspace, data.path))
: actual.ScriptService.existsScriptByPath(data),
getScriptByPath: async (data: { workspace: string; path: string }) => {
getScriptByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByPath(data.workspace, data.path)
// `getDraft` mirrors production's overlay: the row plus the caller's
// draft and a `no_deployed` marker (draft-only item). The diff tool
// reads through this shape — without it every draft looks absent.
const draft = data.getDraft
? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'script', path: data.path })
: null
if (!script) {
throw new Error(`Script "${data.path}" not found in benchmark workspace`)
if (data.getDraft && draft) {
return {
...(draft.value as Record<string, unknown>),
path: data.path,
draft: draft.value,
no_deployed: true
}
}
throw Object.assign(
new Error(`Script "${data.path}" not found in benchmark workspace`),
{ status: 404 }
)
}
return script
return data.getDraft
? { ...script, draft: draft?.value ?? undefined, no_deployed: false }
: script
}
return actual.ScriptService.getScriptByPath(data)
},
@@ -166,13 +185,30 @@ vi.mock('$lib/gen', async () => {
hasBenchmarkWorkspace(data.workspace)
? Boolean(getBenchmarkFlowByPath(data.workspace, data.path))
: actual.FlowService.existsFlowByPath(data),
getFlowByPath: async (data: { workspace: string; path: string }) => {
getFlowByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
// Mirror production's `getDraft` overlay (see getScriptByPath above).
const draft = data.getDraft
? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'flow', path: data.path })
: null
if (!flow) {
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
if (data.getDraft && draft) {
return {
...(draft.value as Record<string, unknown>),
path: data.path,
draft: draft.value,
no_deployed: true
}
}
throw Object.assign(
new Error(`Flow "${data.path}" not found in benchmark workspace`),
{ status: 404 }
)
}
return flow
return data.getDraft
? { ...flow, draft: draft?.value ?? undefined, no_deployed: false }
: flow
}
return actual.FlowService.getFlowByPath(data)
},
@@ -341,13 +377,37 @@ vi.mock('$lib/gen', async () => {
hasBenchmarkWorkspace(data.workspace)
? (listBenchmarkApps(data.workspace) ?? [])
: actual.AppService.listApps(data),
getAppByPath: async (data: { workspace: string; path: string }) => {
getAppByPath: async (data: {
workspace: string
path: string
getDraft?: boolean
rawApp?: boolean
}) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const app = getBenchmarkAppByPath(data.workspace, data.path)
// Mirror production's `getDraft` overlay (see getScriptByPath above).
// Benchmark app drafts live under the raw_app kind.
const draft = data.getDraft
? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'raw_app', path: data.path })
: null
if (!app) {
throw new Error(`App "${data.path}" not found in benchmark workspace`)
if (data.getDraft && draft) {
return {
...(draft.value as Record<string, unknown>),
path: data.path,
raw_app: true,
draft: draft.value,
no_deployed: true
}
}
throw Object.assign(
new Error(`App "${data.path}" not found in benchmark workspace`),
{ status: 404 }
)
}
return app
return data.getDraft
? { ...app, draft: draft?.value ?? undefined, no_deployed: false }
: app
}
return actual.AppService.getAppByPath(data)
}
ai_evals/cases/flow.yaml
+82
View File
@@ -480,3 +480,85 @@
judgeChecklist:
- "the flow includes a final top-level step named `webhook_response`"
- "`webhook_response` returns `ok: true` and the order summary"
- id: flow-test17-implicit-schedule-intent
prompt: |-
I want this order processing flow to run on its own every morning at 07:30 UTC.
Set that up for me. Do not ask me for the flow path.
initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json
toolExpect:
requiredToolsUsed:
- create_schedule
toolCallArgs:
- tool: create_schedule
field: path
stringStartsWithAnyOf:
- f/
- u/
stringMustNotStartWithAnyOf:
- schedules/
- tool: create_schedule
field: schedule
stringIncludesAnyOf:
- 30 7
- tool: create_schedule
field: timezone
stringIncludesAnyOf:
- UTC
skipJudge: true
judgeChecklist:
- "a schedule is created for the flow that runs daily at 07:30 UTC"
- id: flow-test18-implicit-http-trigger-intent
prompt: |-
I need to be able to kick off this order processing flow by sending it an HTTP POST
from an external system, with no authentication. Use route path `ai-evals/order-processing-implicit`.
Do not ask me for the flow path.
initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json
toolExpect:
requiredToolsUsed:
- create_trigger
toolCallArgs:
- tool: create_trigger
field: path
stringStartsWithAnyOf:
- f/
- u/
stringMustNotStartWithAnyOf:
- schedules/
- tool: create_trigger
field: kind
stringStartsWithAnyOf:
- http
- tool: create_trigger
field: config.http_method
stringIncludesAnyOf:
- post
- tool: create_trigger
field: config.authentication_method
stringIncludesAnyOf:
- none
- tool: create_trigger
field: config.route_path
stringIncludesAnyOf:
- ai-evals/order-processing-implicit
skipJudge: true
judgeChecklist:
- "an HTTP trigger is created for the flow that accepts unauthenticated POST requests"
- id: flow-test19-implicit-email-trigger-intent
prompt: |-
Make this order processing flow run automatically whenever an email is received.
Do not ask me for the flow path.
initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json
toolExpect:
requiredToolsUsed:
- create_trigger
toolCallArgs:
- tool: create_trigger
field: kind
stringStartsWithAnyOf:
- email
skipJudge: true
judgeChecklist:
- "an email trigger (kind email) is created, or the user is told how to enable email triggering on the instance"
ai_evals/cases/global.yaml
+173 -3
View File
@@ -1484,9 +1484,15 @@
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
# A pipeline node is DECLARATIVE: triggers are declared by `-- on <ref>`
# annotations (the trigger row is created separately) and a `-- materialize`
# output is a MANAGED write where the body is a bare SELECT that the runtime
# wraps in the create/replace. Do not expect a separate trigger config or a
# hand-written CREATE TABLE / INSERT — those would be wrong for a materialize node.
- builds a data pipeline node as a script (not a flow)
- marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`)
- declares a schedule trigger and writes its output to a managed DuckLake table
- declares the schedule trigger with the `-- on schedule` annotation comment (this annotation is the correct and complete way a pipeline node binds a schedule; no separate trigger configuration is expected)
- declares the managed DuckLake output with `-- materialize ducklake://<table>` and writes the body as a bare SELECT (materialize is a managed write, so the node correctly does NOT hand-write its own CREATE TABLE / INSERT)
- leaves the result as an AI draft and does not deploy or save it
- id: global-test-pipeline-two-node-chain
@@ -1500,6 +1506,12 @@
maxTurns: 14
validate:
draftCountAtLeast: 2
requiredDrafts:
- type: script
pathStartsWith: f/evals/global/
valueIncludes:
- pipeline
- ducklake
forbiddenDrafts:
- type: flow
pathStartsWith: f/evals/global/
@@ -1511,12 +1523,65 @@
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
# Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers and
# `-- materialize ducklake://<table>` is a managed write whose body is a bare
# SELECT. Do not expect hand-written CREATE TABLE / INSERT on a materialize node.
- creates two data pipeline nodes as scripts (not a flow) in f/evals/global
- both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
- the first ingests orders into a DuckLake table
- the second reads that same table and writes a daily rollup, wired to the first step's output asset
- the first ingests orders into a DuckLake table (a `-- materialize ducklake://<table>` output with a bare SELECT body is correct; no hand-written CREATE TABLE / INSERT is expected)
- the second reads that same table via `-- on ducklake://<that-table>` and materializes a daily rollup table, wiring it to the first step's output asset
- leaves both as AI drafts without deploying
- id: global-test-pipeline-complex-incremental
prompt: |-
Build a data pipeline in the `f/evals/global` folder for our web shop's
orders. It has three steps:
1. On a schedule, ingest the raw order CSVs under `s3://raw/orders/` into a
managed DuckLake table.
2. An incremental daily rollup: read that raw orders table and, on each run,
append just the current day's order count and total revenue into a second
DuckLake table. It should process one day at a time, not rebuild the whole
table every run.
3. A final step that reads the daily rollup table and exports the latest data
as a Parquet file to `s3://reports/` for the BI team.
Wire each step to the previous step's output so they form one pipeline. Keep
everything as AI drafts — don't deploy or save.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 18
validate:
draftCountAtLeast: 3
requiredDrafts:
- type: script
pathStartsWith: f/evals/global/
valueIncludes:
- pipeline
- ducklake
forbiddenDrafts:
- type: flow
pathStartsWith: f/evals/global/
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- write_flow
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
# Pipeline nodes are declarative: `-- on <ref>` binds inputs/triggers, and a
# DuckLake `-- materialize` output is a managed write whose body is a bare SELECT
# (the runtime performs the create/replace/append/merge). Do not expect a
# separate trigger config or hand-written CREATE TABLE / INSERT on a
# materialize node. S3/Parquet output is NOT materialize: the body writes it.
- builds the pipeline as three independent scripts (not a flow) in f/evals/global
- every node carries the pipeline annotation in its own comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
- step 1 binds a schedule with `-- on schedule` and declares a managed DuckLake output with `-- materialize ducklake://<table>` and a bare SELECT body (no separate trigger config or hand-written CREATE TABLE is expected)
- "step 2 is incremental: each run adds only that day's rows to a second DuckLake table rather than rebuilding the whole table every run (e.g. an `append` or `key=<col>` merge materialize mode, not a full replace). Selecting the day via the `-- partitioned daily` + `{partition}` / `wm_partition(...)` idiom is the idiomatic form, but an equivalent current-day filter also satisfies this; a full-refresh/replace of the whole table does not"
- step 2 reads the same DuckLake table step 1 writes (via `-- on ducklake://<that-table>`), wiring it to step 1's output asset
- step 3 reads the daily rollup table and exports it as a Parquet file to S3
- does not misuse `-- materialize` for the S3 Parquet export (materialize is DuckLake-only; the S3 output is written by the script body, e.g. a DuckDB COPY or an SDK write)
- leaves all three nodes as AI drafts without deploying or saving
- id: global-path5-create-folder-then-draft
prompt: |-
Create a new shared folder called "analytics" for our data work, then draft a
@@ -1707,3 +1772,108 @@
skipJudge: true
judgeChecklist:
- deletes the deployed script via delete_workspace_item rather than a raw API endpoint
- id: global-draft-diff-report
prompt: |-
Update the existing workspace script at `f/evals/global/format_greeting` so the returned message ends with an exclamation mark, keeping everything else the same.
Then show me exactly what your draft changes compared to the deployed version.
Leave the result as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/format_greeting
language: bun
valueIncludes:
- "!"
toolExpect:
requiredToolsUsed:
- diff
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: diff
field: path
stringIncludesAnyOf:
- f/evals/global/format_greeting
judgeChecklist:
- creates an AI draft for the existing f/evals/global/format_greeting script with the exclamation-mark change
- the draft changes only the returned message's punctuation — summary, language, path, and the rest of the code are untouched
- does not deploy or save the draft to the workspace
- id: global-resource-manual-credentials
prompt: |-
Set up a resource for our production Postgres database at `f/evals/global/prod_db` (host db.internal.example.com, port 5432, database `orders`, user `app`).
I don't want to paste the password into this chat — prepare everything so I can enter it myself.
Leave it as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 12
validate:
draftCountAtLeast: 1
requiredDrafts:
- type: resource
path: f/evals/global/prod_db
valueIncludes:
- db.internal.example.com
- orders
toolExpect:
requiredToolsUsed:
- write_resource
- open_page
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
# The model may land the user in the resource's drawer or in the drawer of
# the secret variable it created for the password — both are correct.
- tool: open_page
field: page
stringIncludesAnyOf:
- resources
- variables
- tool: open_page
field: open
stringIncludesAnyOf:
- prod_db
- password
judgeChecklist:
- creates a postgres resource draft at f/evals/global/prod_db with the provided host, port, database, and user
- the password is left for the user to provide (empty, a placeholder, or a secret variable reference) — no invented password value presented as real
- does not deploy or save anything to the workspace
- id: global-test29-email-trigger-draft
prompt: |-
Set up a draft auto-reply job.
Create a Bun script at `f/evals/global/email_pong` that returns the string "pong".
Then set it up so it runs whenever an email is received at the inbox `pong`.
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
runtime:
maxTurns: 10
validate:
requiredDrafts:
- type: script
path: f/evals/global/email_pong
language: bun
valueIncludes:
- pong
toolExpect:
requiredToolsUsed:
- write_script
- write_trigger
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
# "Runs when an email is received" must resolve to the native email trigger kind,
# never a faked HTTP webhook. Assert the recorded tool-call kind (not the draft):
# it holds even on a CE backend where email trigger routes (smtp+private) 404.
- tool: write_trigger
field: kind
stringIncludesAnyOf:
- email
skipJudge: true
backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json
Generated
-31
View File
@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Jsonb",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c"
}
backend/Cargo.lock
Generated
+114 -112
View File
@@ -879,7 +879,7 @@ checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -2290,9 +2290,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.3.0"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -2410,9 +2410,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.6.3"
version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776"
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
dependencies = [
"clap_builder",
"clap_derive",
@@ -2432,14 +2432,14 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.6.3"
version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077"
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.119",
"syn 3.0.3",
]
[[package]]
@@ -5510,9 +5510,9 @@ dependencies = [
[[package]]
name = "glob"
version = "0.3.3"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "globset"
@@ -7003,9 +7003,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.186"
version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "libffi-sys"
@@ -8861,9 +8861,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
version = "2.8.7"
version = "2.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9"
checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2"
dependencies = [
"memchr",
"ucd-trie",
@@ -8871,9 +8871,9 @@ dependencies = [
[[package]]
name = "pest_derive"
version = "2.8.7"
version = "2.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58"
checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd"
dependencies = [
"pest",
"pest_generator",
@@ -8881,9 +8881,9 @@ dependencies = [
[[package]]
name = "pest_generator"
version = "2.8.7"
version = "2.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7"
checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6"
dependencies = [
"pest",
"pest_meta",
@@ -8894,9 +8894,9 @@ dependencies = [
[[package]]
name = "pest_meta"
version = "2.8.7"
version = "2.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210"
checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c"
dependencies = [
"pest",
]
@@ -9967,7 +9967,7 @@ checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -10638,9 +10638,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.15.0"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
dependencies = [
"web-time",
"zeroize",
@@ -11133,7 +11133,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -11198,7 +11198,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -11870,9 +11870,9 @@ dependencies = [
[[package]]
name = "sse-stream"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39f24a9b78c40b90817bbcd1821c74ddfd74916aadd29403d001532a9195532d"
checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e"
dependencies = [
"bytes",
"futures-util",
@@ -12410,9 +12410,9 @@ dependencies = [
[[package]]
name = "syn"
version = "3.0.2"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
@@ -12805,7 +12805,7 @@ checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -13172,9 +13172,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
version = "0.1.18"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -13771,7 +13771,7 @@ checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.2",
"syn 3.0.3",
]
[[package]]
@@ -14479,7 +14479,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14564,7 +14564,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"async-stream",
"async-trait",
@@ -14597,7 +14597,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14610,7 +14610,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"argon2",
@@ -14749,7 +14749,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14772,7 +14772,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14787,7 +14787,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14813,7 +14813,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -14823,7 +14823,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14840,7 +14840,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -14862,7 +14862,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14885,7 +14885,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14901,7 +14901,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14922,7 +14922,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14943,7 +14943,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14957,7 +14957,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14992,7 +14992,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15017,7 +15017,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"flate2",
@@ -15035,7 +15035,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15057,7 +15057,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15077,7 +15077,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15114,7 +15114,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15142,7 +15142,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"lazy_static",
"serde",
@@ -15154,7 +15154,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -15179,7 +15179,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15193,7 +15193,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15228,7 +15228,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15242,7 +15242,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15261,7 +15261,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -15305,6 +15305,7 @@ dependencies = [
"lazy_static",
"magic-crypt",
"mail-send",
"memchr",
"native-tls",
"once_cell",
"openidconnect",
@@ -15364,7 +15365,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -15383,7 +15384,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"regex",
"serde",
@@ -15398,7 +15399,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15422,7 +15423,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"futures",
@@ -15439,7 +15440,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15455,7 +15456,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15476,7 +15477,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15507,7 +15508,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -15532,7 +15533,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-stream",
@@ -15566,7 +15567,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"futures",
@@ -15584,7 +15585,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15593,7 +15594,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15605,7 +15606,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15617,7 +15618,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"gosyn",
@@ -15629,7 +15630,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15641,7 +15642,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15653,7 +15654,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -15664,7 +15665,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15675,7 +15676,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15687,7 +15688,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15698,7 +15699,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15720,7 +15721,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -15732,7 +15733,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15746,7 +15747,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15763,7 +15764,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15776,7 +15777,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde",
@@ -15788,7 +15789,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15806,7 +15807,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -15822,7 +15823,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15838,7 +15839,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde",
@@ -15849,7 +15850,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15888,7 +15889,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"const_format",
@@ -15928,7 +15929,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -15939,7 +15940,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15973,7 +15974,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15997,7 +15998,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16030,7 +16031,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-amqp"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16057,7 +16058,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16090,7 +16091,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16110,7 +16111,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16144,7 +16145,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16180,7 +16181,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16203,7 +16204,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16227,7 +16228,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16251,7 +16252,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16286,7 +16287,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16314,7 +16315,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16339,7 +16340,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"bitflags 2.13.1",
@@ -16358,7 +16359,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16378,6 +16379,7 @@ dependencies = [
"dotenv",
"eventsource-stream",
"flume",
"fs4",
"futures",
"gcp_auth",
"git-version",
@@ -16472,7 +16474,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"bytes",
"futures",
backend/Cargo.toml
+10 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.764.0"
version = "1.769.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.764.0"
version = "1.769.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -176,7 +176,7 @@ ruby = ["windmill-worker/ruby"]
rlang = ["windmill-worker/rlang"]
all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"]
# For windows we have another set of languages enabled
all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"]
all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"]
# Edition meta-features: shared groups
run_inline = ["windmill-api/run_inline"]
oss_core = [
@@ -200,7 +200,12 @@ ce = ["ce_rpi", "jemalloc", "dind", "agent_worker_server"]
# Edition meta-features: EE variants
ee = ["ce", "ee_core", "ee_server", "kafka-gssapi"]
ee_rhel = ["ce_core", "ee_core", "kafka-gssapi", "all_languages"]
ee_windows = ["ce_core", "ee_core", "all_languages_windows"]
# The Windows binary is worker-only, but a non-agent worker runs windmill-api on
# localhost (main.rs run_server, `if !is_agent`) and jobs call back into it, so
# it needs every feature its own plumbing or its jobs invoke in-process; drop
# only external/server-facing surface the worker never runs.
worker_windows_core = ["private", "operator", "parquet", "quickjs", "enterprise", "prometheus", "otel", "jemalloc", "windmill-worker/mcp", "windmill-store/mcp", "windmill-worker/bedrock", "openidconnect", "run_inline", "windmill-api/instance_smtp", "oauth2"]
ee_windows = ["worker_windows_core", "all_languages_windows"]
all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embedding", "parquet", "prometheus", "flow_testing",
"openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "kafka-gssapi", "nats", "otel", "dind", "websocket", "http_trigger",
"postgres_trigger", "mcp", "mqtt_trigger", "amqp_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", "smtp", "stripe",
@@ -583,6 +588,7 @@ rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]}
jsonwebtoken = "8.3.0"
pem = "3.0.1"
nix = { version = "0.27.1", features = ["process", "signal"] }
fs4 = "0.13"
tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" }
hf-hub = "0.4.3"
tokenizers = "0.14.1"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
c6287e2e71a5ab2e3331c46d8e13ebfaeff11ac6
a9e0af17f4c972f9866f7dd1925aedd2b3b27052
backend/migrations/20260716152346_custom_instance_replication_user.down.sql
+14
View File
@@ -0,0 +1,14 @@
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN
ALTER ROLE custom_instance_user REPLICATION;
END IF;
DROP ROLE IF EXISTS custom_instance_replication_user;
DELETE FROM global_settings WHERE name = 'custom_instance_replication_pwd';
EXCEPTION
WHEN others THEN
RAISE NOTICE 'custom_instance_replication_user down-migration error, skipping: %', SQLERRM;
END
$$;
backend/migrations/20260716152346_custom_instance_replication_user.up.sql
+34
View File
@@ -0,0 +1,34 @@
-- Dedicated logical-replication role used by postgres triggers on custom-instance
-- datatables. Its password is stored server-only in global_settings.custom_instance_replication_pwd
-- (hidden from the config surface); membership in custom_instance_user lets it manage
-- publications on the datatable tables. custom_instance_user itself must not hold REPLICATION.
DO $$
DECLARE
pwd text;
BEGIN
SELECT gen_random_uuid()::text INTO pwd;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_replication_user') THEN
EXECUTE format('ALTER USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd);
ELSE
EXECUTE format('CREATE USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd);
END IF;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN
GRANT custom_instance_user TO custom_instance_replication_user;
ALTER ROLE custom_instance_user NOREPLICATION;
END IF;
INSERT INTO global_settings (name, value)
VALUES ('custom_instance_replication_pwd', to_jsonb(pwd::text))
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;
-- Drop any replication password an earlier iteration stored in the operator-facing row.
UPDATE global_settings
SET value = value - 'replication_user_pwd'
WHERE name = 'custom_instance_pg_databases';
EXCEPTION
WHEN others THEN
RAISE NOTICE 'custom_instance_replication_user migration error, skipping: %', SQLERRM;
END
$$;
backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Irreversible data repair: once the leading slash is restored, the rows are
-- indistinguishable from paths that always had it. Intentionally a no-op.
SELECT 1;
backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql
+144
View File
@@ -0,0 +1,144 @@
-- Repair s3object asset paths recorded without their default-storage leading
-- slash. An S3 asset path is `<storage>/<key>` with an empty storage segment
-- (leading slash) for the workspace default: `s3:///exports/x` -> `/exports/x`.
-- Between 2026-07-06 (#9939) and the parser fix, the asset parser stripped
-- leading slashes, so default-storage assets were recorded as `exports/x` —
-- indistinguishable from a secondary storage named `exports`. For rows created
-- in that window (cutoff one day early for safety), a slashless path whose
-- first segment is NOT a storage name — neither a configured secondary storage
-- nor the reserved `_default_` alias — can only be a default-storage key, so it
-- gets its slash back. Rows already starting with `/` are always correct.
--
-- Best-effort by nature: the corruption itself conflated a stripped default key
-- with a named ref, so identity is inferred from the storage config AS IT IS NOW.
-- A named ref to a storage that was since removed/renamed (or never configured)
-- is the one residual false-positive — it would be repaired as if default. The
-- `created_at` window bounds this for `asset`; `script_trigger` has no timestamp
-- and relies on the storage-name heuristic alone. Both are acceptable given how
-- rare mid-window storage churn is versus the common default-key case this fixes.
--
-- Same corruption hit `script_trigger.trigger_ref` (the pipeline cascade edges,
-- stored as `s3://<path>`): a default-storage edge recorded as `s3://exports/x`
-- instead of `s3:///exports/x` no longer matches the producer's post-fix write
-- ref at dispatch (asset_dispatch rebuilds `s3://` + the repaired asset path and
-- does an exact `trigger_ref =` match), silently breaking the edge. Repaired with
-- the same storage-name heuristic — script_trigger has no created_at, but a
-- correct default ref is always `s3:///…` and a correct named ref always leads
-- with a real storage name, so a `s3://<seg>/…` ref whose seg isn't a storage is
-- unambiguously a slash-stripped default-storage ref.
--
-- `join_pending_inputs.trigger_ref` (the AND-join barrier) is deliberately NOT
-- repaired: it is transient slot state cleared on fire, so a window-era `s3://…`
-- slot is superseded once inputs re-arrive under the corrected ref (and deleting
-- live slots could drop an in-flight accumulation). materialized_asset_schema is
-- unaffected — it only ever holds ducklake asset_kind, never s3object.
--
-- Data-repair only: wrapped so a failure NOTICEs and never blocks the release.
DO $migration$
BEGIN
CREATE TEMP TABLE __asset_slash_fix_cache (
workspace_id TEXT PRIMARY KEY,
names TEXT[] NOT NULL
) ON COMMIT DROP;
-- Reserved first-path-segments that denote a real storage (so a slashless
-- path leading with one is a genuine named ref, NOT a slash-stripped default
-- key): the workspace's secondary_storage names PLUS `_default_`, the alias
-- the runtime treats as the primary storage (workspaces.rs fork_storage_ref).
-- `s3://_default_/key` is a valid explicit-default ref recorded verbatim as
-- `_default_/key`; prepending a slash would corrupt it to key `_default_/key`.
-- The JSON is parsed at most once per workspace (candidate assets can repeat
-- a workspace millions of times via job usages), and only workspaces that
-- actually have candidate rows are ever fetched.
CREATE FUNCTION pg_temp.__asset_slash_fix_storages(ws TEXT) RETURNS TEXT[] AS $fn$
DECLARE
result TEXT[];
BEGIN
SELECT c.names INTO result FROM __asset_slash_fix_cache c WHERE c.workspace_id = ws;
IF FOUND THEN
RETURN result;
END IF;
SELECT ARRAY['_default_'] || COALESCE(array_agg(k), '{}') INTO result
FROM workspace_settings s
CROSS JOIN LATERAL jsonb_object_keys(
CASE WHEN jsonb_typeof(s.large_file_storage -> 'secondary_storage') = 'object'
THEN s.large_file_storage -> 'secondary_storage'
ELSE '{}'::JSONB END
) k
WHERE s.workspace_id = ws;
result := COALESCE(result, ARRAY['_default_']);
INSERT INTO __asset_slash_fix_cache VALUES (ws, result);
RETURN result;
END
$fn$ LANGUAGE plpgsql;
-- Duplicates first: when the corrected `/path` row already exists for the
-- same usage (recorded before the regression, or re-recorded after the
-- parser fix), prepending the slash would violate the primary key
-- (workspace_id, path, kind, usage_path, usage_kind) — drop the slashless
-- duplicate instead.
DELETE FROM asset a
WHERE a.kind = 's3object'
AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ
AND a.path NOT LIKE '/%'
AND a.path <> ''
AND length(a.path) < 255
AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id))
AND EXISTS (
SELECT 1 FROM asset b
WHERE b.workspace_id = a.workspace_id
AND b.path = '/' || a.path
AND b.kind = a.kind
AND b.usage_path = a.usage_path
AND b.usage_kind = a.usage_kind
);
-- length < 255 keeps the prepend within the VARCHAR(255) column; a 255-char
-- slashless path cannot be repaired and is left as-is rather than erroring.
UPDATE asset a
SET path = '/' || a.path
WHERE a.kind = 's3object'
AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ
AND a.path NOT LIKE '/%'
AND a.path <> ''
AND length(a.path) < 255
AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id));
-- script_trigger.trigger_ref for asset edges is `s3://<path>`. A corrupted
-- default-storage edge reads `s3://<seg>/…` (exactly two slashes); a correct
-- default ref is `s3:///…` and is excluded by the NOT LIKE. `substring(from 6)`
-- is the `<path>` after the `s3://` prefix. Delete a slashless edge whose
-- corrected twin already exists for the same runnable (fetch_subscribers has
-- no DISTINCT, so a duplicate would double-dispatch the subscriber).
DELETE FROM script_trigger a
WHERE a.trigger_kind = 'asset'
AND a.trigger_ref LIKE 's3://%'
AND a.trigger_ref NOT LIKE 's3:///%'
AND substring(a.trigger_ref FROM 6) <> ''
AND split_part(substring(a.trigger_ref FROM 6), '/', 1)
<> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id))
AND EXISTS (
SELECT 1 FROM script_trigger b
WHERE b.workspace_id = a.workspace_id
AND b.runnable_kind = a.runnable_kind
AND b.runnable_path = a.runnable_path
AND b.trigger_kind = a.trigger_kind
AND b.trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6)
);
UPDATE script_trigger a
SET trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6)
WHERE a.trigger_kind = 'asset'
AND a.trigger_ref LIKE 's3://%'
AND a.trigger_ref NOT LIKE 's3:///%'
AND substring(a.trigger_ref FROM 6) <> ''
AND split_part(substring(a.trigger_ref FROM 6), '/', 1)
<> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id));
-- The temp table is ON COMMIT DROP; drop the function too so nothing
-- lingers on a pooled connection.
DROP FUNCTION pg_temp.__asset_slash_fix_storages(TEXT);
EXCEPTION WHEN OTHERS THEN
RAISE NOTICE 'skipping s3 asset leading-slash repair: %', SQLERRM;
END
$migration$;
backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE workspace_settings
DROP COLUMN IF EXISTS error_handler_fallback_to_instance_alerts;
backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE workspace_settings
ADD COLUMN IF NOT EXISTS error_handler_fallback_to_instance_alerts BOOLEAN NOT NULL DEFAULT false;
backend/parsers/windmill-parser-py-asset/src/lib.rs
+17 -16
View File
@@ -383,7 +383,7 @@ def main():
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "test.csv".to_string(),
path: "/test.csv".to_string(),
access_type: Some(R),
columns: None,
},])
@@ -441,7 +441,7 @@ def main():
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "analytics/x.csv".to_string(),
path: "/analytics/x.csv".to_string(),
access_type: Some(W),
columns: None,
},])
@@ -450,10 +450,11 @@ def main():
#[test]
fn test_py_write_key_matches_duckdb_read_key() {
// Cross-language lineage: this write records `exports/x`, the same path a
// DuckDB `read_csv('s3://exports/x')` resolves to (see
// windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`),
// so the producer and consumer connect in the pipeline graph.
// Cross-language lineage: this default-storage write records
// `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')`
// resolves to (see windmill-parser-sql-asset
// `test_duckdb_read_key_matches_sdk_write_key`), so the producer and
// consumer connect in the pipeline graph.
let input = r#"
import wmill
from wmill import S3Object
@@ -465,7 +466,7 @@ def main():
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "exports/x".to_string(),
path: "/exports/x".to_string(),
access_type: Some(W),
columns: None,
},])
@@ -508,7 +509,7 @@ def main():
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "dir/in.csv".to_string(),
path: "/dir/in.csv".to_string(),
access_type: Some(R),
columns: None,
},])
@@ -531,14 +532,14 @@ def main():
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "mybucket/dir/in.csv".to_string(),
access_type: Some(R),
path: "/out.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "out.json".to_string(),
access_type: Some(W),
path: "mybucket/dir/in.csv".to_string(),
access_type: Some(R),
columns: None,
},
])
@@ -564,25 +565,25 @@ def main():
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/enriched.json".to_string(),
path: "/pipelines/km_real/enriched.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/raw_events.json".to_string(),
path: "/pipelines/km_real/raw_events.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/report.json".to_string(),
path: "/pipelines/km_real/report.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/summary.json".to_string(),
path: "/pipelines/km_real/summary.json".to_string(),
access_type: Some(W),
columns: None,
},
backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs
+43 -33
View File
@@ -1261,13 +1261,13 @@ mod tests {
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "a.parquet".to_string(),
path: "/a.parquet".to_string(),
access_type: Some(R),
columns: None
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "c.parquet".to_string(),
path: "/c.parquet".to_string(),
access_type: Some(W),
columns: None
},
@@ -1284,25 +1284,35 @@ mod tests {
#[test]
fn test_duckdb_read_key_matches_sdk_write_key() {
// Cross-language lineage: a TS `writeS3File({ s3: "exports/x" })` or
// Python `write_s3_file(S3Object(s3="exports/x"))` records the asset path
// `exports/x` (default storage). A DuckDB reader of the same object must
// resolve to the identical path so the graph connects the producer and
// consumer — both the bare `s3://exports/x` and the triple-slash
// `s3:///exports/x` default-storage form must yield `exports/x`.
for uri in ["s3://exports/x", "s3:///exports/x"] {
let input = format!("SELECT * FROM read_csv('{uri}');");
let assets = parse_assets(&input).expect("parse").assets;
assert_eq!(
assets,
vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "exports/x".to_string(),
access_type: Some(R),
columns: None
}],
"DuckDB read of {uri} must resolve to the SDK write key"
);
}
// Python `write_s3_file(S3Object(s3="exports/x"))` records the asset
// path `/exports/x` (default storage, leading slash). A DuckDB reader
// of the same object uses the triple-slash default-storage URI and
// must resolve to the identical path so the graph connects producer
// and consumer. The bare `s3://exports/x` form names storage
// `exports` instead — a different object, a different path.
let input = "SELECT * FROM read_csv('s3:///exports/x');";
let assets = parse_assets(input).expect("parse").assets;
assert_eq!(
assets,
vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/exports/x".to_string(),
access_type: Some(R),
columns: None
}],
);
let input = "SELECT * FROM read_csv('s3://exports/x');";
let assets = parse_assets(input).expect("parse").assets;
assert_eq!(
assets,
vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "exports/x".to_string(),
access_type: Some(R),
columns: None
}],
);
}
#[test]
@@ -1318,7 +1328,7 @@ mod tests {
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "out.csv".to_string(),
path: "/out.csv".to_string(),
access_type: Some(W),
columns: None
}])
@@ -1335,7 +1345,7 @@ mod tests {
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "referenced.csv".to_string(),
path: "/referenced.csv".to_string(),
access_type: Some(R),
columns: None
}])
@@ -1355,7 +1365,7 @@ mod tests {
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "data.csv".to_string(),
path: "/data.csv".to_string(),
access_type: Some(RW),
columns: None
}])
@@ -1375,7 +1385,7 @@ mod tests {
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "data.parquet".to_string(),
path: "/data.parquet".to_string(),
access_type: Some(RW),
columns: None
}])
@@ -1393,13 +1403,13 @@ mod tests {
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "a.parquet".to_string(),
path: "/a.parquet".to_string(),
access_type: Some(R),
columns: None
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "b.parquet".to_string(),
path: "/b.parquet".to_string(),
access_type: Some(R),
columns: None
}
@@ -1419,7 +1429,7 @@ mod tests {
s.map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "data.parquet".to_string(),
path: "/data.parquet".to_string(),
access_type: Some(RW),
columns: None
}])
@@ -2101,7 +2111,7 @@ mod tests {
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "example_file.parquet");
assert_eq!(result[0].path, "/example_file.parquet");
assert_eq!(result[0].access_type, Some(R));
let columns = result[0].columns.as_ref().expect("Should have columns");
@@ -2132,7 +2142,7 @@ mod tests {
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "example_file.parquet");
assert_eq!(result[0].path, "/example_file.parquet");
assert!(result[0].columns.is_none());
}
@@ -2145,7 +2155,7 @@ mod tests {
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "example_file.parquet");
assert_eq!(result[0].path, "/example_file.parquet");
let columns = result[0].columns.as_ref().expect("Should have columns");
assert_eq!(columns.get("col1"), Some(&R));
@@ -2164,7 +2174,7 @@ mod tests {
assert_eq!(result.len(), 2);
assert!(result.iter().any(|a| {
a.path == "file1.parquet"
a.path == "/file1.parquet"
&& a.columns.as_ref().map_or(false, |c| c.contains_key("col1"))
}));
assert!(result.iter().any(|a| {
@@ -2197,7 +2207,7 @@ mod tests {
assert_eq!(result.len(), 1);
assert_eq!(result[0].kind, AssetKind::S3Object);
assert_eq!(result[0].path, "test.parquet");
assert_eq!(result[0].path, "/test.parquet");
assert_eq!(result[0].access_type, Some(R));
let columns = result[0].columns.as_ref().expect("Should have columns");
backend/parsers/windmill-parser-ts-asset/src/lib.rs
+13 -12
View File
@@ -433,7 +433,7 @@ mod tests {
s.map(|r| r.assets).map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "test.csv".to_string(),
path: "/test.csv".to_string(),
access_type: Some(R),
columns: None,
},])
@@ -461,7 +461,7 @@ mod tests {
s.map(|r| r.assets).map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/raw_events.json".to_string(),
path: "/pipelines/km_real/raw_events.json".to_string(),
access_type: Some(W),
columns: None,
},])
@@ -470,10 +470,11 @@ mod tests {
#[test]
fn test_ts_write_key_matches_duckdb_read_key() {
// Cross-language lineage: this write records `exports/x`, the same path a
// DuckDB `read_csv('s3://exports/x')` resolves to (see
// windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`),
// so the producer and consumer connect in the pipeline graph.
// Cross-language lineage: this default-storage write records
// `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')`
// resolves to (see windmill-parser-sql-asset
// `test_duckdb_read_key_matches_sdk_write_key`), so the producer and
// consumer connect in the pipeline graph.
let input = r#"
import * as wmill from "windmill-client"
export async function main() {
@@ -485,7 +486,7 @@ mod tests {
s.map(|r| r.assets).map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "exports/x".to_string(),
path: "/exports/x".to_string(),
access_type: Some(W),
columns: None,
},])
@@ -570,25 +571,25 @@ mod tests {
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/enriched.json".to_string(),
path: "/pipelines/km_real/enriched.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/raw_events.json".to_string(),
path: "/pipelines/km_real/raw_events.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/report.json".to_string(),
path: "/pipelines/km_real/report.json".to_string(),
access_type: Some(W),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "pipelines/km_real/summary.json".to_string(),
path: "/pipelines/km_real/summary.json".to_string(),
access_type: Some(W),
columns: None,
},
@@ -609,7 +610,7 @@ mod tests {
s.map(|r| r.assets).map_err(|e| e.to_string()),
Ok(vec![ParseAssetsResult {
kind: AssetKind::S3Object,
path: "out.json".to_string(),
path: "/out.json".to_string(),
access_type: Some(W),
columns: None,
},])
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6272,7 +6272,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"convert_case",
"serde",
@@ -6293,7 +6293,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6305,7 +6305,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6317,7 +6317,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6329,7 +6329,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6341,7 +6341,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6353,7 +6353,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6364,7 +6364,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6375,7 +6375,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6387,7 +6387,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6398,7 +6398,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6420,7 +6420,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6432,7 +6432,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6446,7 +6446,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6463,7 +6463,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde",
@@ -6488,7 +6488,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6522,7 +6522,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6538,7 +6538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6570,7 +6570,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"serde",
@@ -6581,7 +6581,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.764.0"
version = "1.769.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.764.0"
version = "1.769.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/parsers/windmill-parser/src/asset_parser.rs
+31 -73
View File
@@ -722,26 +722,13 @@ pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(Asset
}
for (prefix, kind) in ASSET_KINDS.iter() {
if s.starts_with(prefix) {
let path = &s[prefix.len()..];
// Canonicalize S3 keys to a single asset identity. The SDK object
// form (`{ s3: "key" }` / `S3Object(s3="key")`, default storage)
// resolves to `s3:///key`, whose path is `/key`, while DuckDB
// `s3://key` and `// on s3://key` yield the bare `key`. Strip every
// leading slash so the triple-slash default-storage form and the
// `s3://storage/key` form share one path — otherwise a TS/Python
// writer and a DuckDB reader of the same object become disconnected
// nodes in the pipeline graph. Stripping ALL leading slashes (not
// just one) keeps the identity stable through URI reconstruction:
// `trigger_spec_to_row` rebuilds `s3://<path>`, so a canonical path
// must never itself start with `/` or the rebuilt ref would parse
// back to a different key. Only leading slashes are touched, so
// Hive-partition keys (`s3://b/y=2024/f.parquet`) are untouched.
let path = if matches!(kind, AssetKind::S3Object) {
path.trim_start_matches('/')
} else {
path
};
return Some((*kind, path));
// The suffix is kept verbatim. For S3 the path encodes the storage:
// `s3://<storage>/<key>`, with an EMPTY storage segment for the
// workspace default — so `s3:///key` yields `/key` (leading slash
// significant, default storage) while `s3://secondary/key` yields
// `secondary/key`. Stripping leading slashes here would conflate a
// default-storage object with a named-storage one.
return Some((*kind, &s[prefix.len()..]));
}
}
None
@@ -1560,52 +1547,43 @@ mod pipeline_annotation_tests {
use super::*;
#[test]
fn s3_key_normalization_unifies_uri_forms() {
// A TS/Python SDK write of `{ s3: "exports/x" }` (default storage)
// resolves to the URI `s3:///exports/x`, while a DuckDB read of
// `s3://exports/x` and the `// on s3://exports/x` trigger form yield the
// bare `exports/x`. All three must canonicalize to one asset key so
// the writer and reader connect in the pipeline graph.
let sdk_write = parse_asset_syntax("s3:///exports/x", false);
let duckdb_read = parse_asset_syntax("s3://exports/x", false);
assert_eq!(sdk_write, Some((AssetKind::S3Object, "exports/x")));
assert_eq!(duckdb_read, Some((AssetKind::S3Object, "exports/x")));
assert_eq!(sdk_write, duckdb_read);
fn s3_path_keeps_storage_distinction() {
// An S3 asset path is `<storage>/<key>` with an empty storage segment
// for the workspace default. The default-storage form `s3:///key`
// yields `/key` (leading slash significant); the named-storage form
// `s3://secondary/key` yields `secondary/key`. The two name DIFFERENT
// objects and must never collapse to one identity.
assert_eq!(
parse_asset_syntax("s3:///exports/x", false),
Some((AssetKind::S3Object, "/exports/x"))
);
assert_eq!(
parse_asset_syntax("s3://exports/x", false),
Some((AssetKind::S3Object, "exports/x"))
);
assert_ne!(
parse_asset_syntax("s3:///exports/x", false),
parse_asset_syntax("s3://exports/x", false)
);
// The `// on` trigger annotation goes through the same function.
assert_eq!(
parse_asset_syntax("s3:///exports/x", true),
parse_asset_syntax("s3://exports/x", true)
Some((AssetKind::S3Object, "/exports/x"))
);
// Explicit-storage form is unaffected (no leading slash to strip).
assert_eq!(
parse_asset_syntax("s3://mybucket/exports/x", false),
Some((AssetKind::S3Object, "mybucket/exports/x"))
parse_asset_syntax("s3://secondary_storage/path/to/file.csv", false),
Some((AssetKind::S3Object, "secondary_storage/path/to/file.csv"))
);
// Hive-partition keys and nested paths under default storage are
// preserved verbatim (only leading slashes are stripped).
// Hive-partition keys are preserved verbatim.
assert_eq!(
parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false),
Some((AssetKind::S3Object, "t/year=2024/month=01/f.parquet"))
Some((AssetKind::S3Object, "/t/year=2024/month=01/f.parquet"))
);
// Every leading slash is stripped so a canonical S3 path never starts
// with `/`. `S3Object(s3="/x")` resolves to the quad-slash URI
// `s3:////x`; the identity must be the bare `x` (not `/x`) so the ref
// that `trigger_spec_to_row` rebuilds round-trips back to it.
assert_eq!(
parse_asset_syntax("s3:////x", false),
Some((AssetKind::S3Object, "x"))
);
assert_eq!(
parse_asset_syntax("s3://///deep///", false),
Some((AssetKind::S3Object, "deep///"))
);
// Non-S3 kinds keep their leading slash (their paths are workspace-
// relative and the slash is significant).
// Non-S3 kinds also keep their suffix verbatim.
assert_eq!(
parse_asset_syntax("res://f/foo", false),
Some((AssetKind::Resource, "f/foo"))
@@ -1616,26 +1594,6 @@ mod pipeline_annotation_tests {
);
}
#[test]
fn s3_explicit_storage_aliases_default_storage_nested_key() {
// Accepted tradeoff of one canonical key: the explicit-storage form
// `s3://storage/key` and the default-storage nested-key form
// `s3:///storage/key` collapse to the same node `storage/key`, even
// though they name different objects. This is a best-effort lineage
// graph that does not split the first segment as a storage name; the
// collision only happens when a storage config is named to match a
// default-storage prefix. Pinned so the aliasing is intentional, not a
// latent surprise.
assert_eq!(
parse_asset_syntax("s3://mybucket/x", false),
parse_asset_syntax("s3:///mybucket/x", false)
);
assert_eq!(
parse_asset_syntax("s3://mybucket/x", false),
Some((AssetKind::S3Object, "mybucket/x"))
);
}
#[test]
fn bare_pipeline_marker() {
let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')");
backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json
Vendored
+5 -5
View File
@@ -741,12 +741,12 @@
}
},
{
"name": "s3 triple-slash default-storage trigger canonicalizes to bare key",
"name": "s3 triple-slash default-storage trigger keeps its leading slash",
"code": "// pipeline\n// on s3:///exports/x\nexport function main() {}",
"expected": {
"in_pipeline": true,
"asset_triggers": [
"s3object:exports/x"
"s3object:/exports/x"
],
"native_triggers": [],
"partition": null,
@@ -756,12 +756,12 @@
}
},
{
"name": "s3 quad-slash trigger strips all leading slashes to the bare key",
"code": "// pipeline\n// on s3:////x\nexport function main() {}",
"name": "s3 named-storage trigger keeps the storage segment",
"code": "// pipeline\n// on s3://secondary_storage/exports/x\nexport function main() {}",
"expected": {
"in_pipeline": true,
"asset_triggers": [
"s3object:x"
"s3object:secondary_storage/exports/x"
],
"native_triggers": [],
"partition": null,
backend/src/monitor.rs
+347 -27
View File
@@ -3193,6 +3193,14 @@ pub async fn monitor_db(
if !initial_load {
verify_license_key(conn.as_sql()).await;
refetch_license_key_if_invalid(conn).await;
// Server-side only: the alert writes to the alerts table and notifies
// the critical channels, so gate it like enforce_offline_caps rather
// than have every worker re-report the same expiry.
if server_mode {
if let Some(db) = conn.as_sql() {
windmill_common::ee_oss::alert_on_online_license_expired(db).await;
}
}
}
};
@@ -4864,12 +4872,95 @@ WHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING c
Ok(())
}
/// Memory usage at a worker's last ping as a fraction of its cgroup limit.
/// Takes the larger of the cgroup-wide reading and the windmill process's
/// jemalloc resident — if only one is present, that value wins; if both are
/// present, the larger is the more conservative (higher-signal) choice.
fn zombie_worker_memory_pct(
usage: Option<i64>,
wm_usage: Option<i64>,
total: Option<i64>,
) -> Option<f64> {
let total = total?;
if total <= 0 {
return None;
}
let used = usage.max(wm_usage)?;
Some(used as f64 / total as f64)
}
struct ZombieFlowCulprit {
worker: String,
ping_at: DateTime<Utc>,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
memory_total: Option<i64>,
worker_group: Option<String>,
worker_instance: Option<String>,
ping_delta_secs: Option<f64>,
}
/// Finds the worker that likely performed and dropped the flow's final state
/// transition when `q.worker` (the outer queue-row worker) looks healthy — a
/// different worker on the same pod/group whose *latest* ping is frozen in the
/// `[last_ping-5s, +15s]` window (a live worker would have advanced its in-place
/// ping past that old window, so a frozen ping there proves it has gone silent),
/// nearest the transition. Diagnostics-only: fails soft to `None`.
async fn find_zombie_flow_culprit_worker(
db: &DB,
q_worker: &str,
last_ping: DateTime<Utc>,
) -> Option<ZombieFlowCulprit> {
let res = sqlx::query_as!(
ZombieFlowCulprit,
r#"
WITH ref AS (
SELECT worker_instance, worker_group FROM worker_ping WHERE worker = $1 LIMIT 1
)
SELECT
wp.worker AS "worker!",
wp.ping_at AS "ping_at!",
wp.memory_usage,
wp.wm_memory_usage,
wp.memory AS memory_total,
wp.worker_group,
wp.worker_instance,
EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))::float8 AS ping_delta_secs
FROM worker_ping wp, ref
WHERE wp.worker <> $1
AND (
(ref.worker_instance IS NOT NULL AND wp.worker_instance = ref.worker_instance)
OR (ref.worker_group IS NOT NULL AND wp.worker_group = ref.worker_group)
)
AND wp.ping_at >= $2::timestamptz - interval '5 seconds'
AND wp.ping_at <= $2::timestamptz + interval '15 seconds'
ORDER BY ABS(EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))) ASC
LIMIT 1
"#,
q_worker,
last_ping,
)
.fetch_optional(db)
.await;
match res {
Ok(culprit) => culprit,
Err(e) => {
tracing::warn!(
"failed to query for zombie-flow culprit worker (q_worker={q_worker}): {e:#}"
);
None
}
}
}
async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
// flow_status is cast ::text on purpose: decoding the jsonb column directly as Box<str>
// yields its binary form (leading version byte) and fails serde_json parsing at column 1.
let flows = sqlx::query!(
r#"
SELECT
j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?",
COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box<str>", r.ping AS last_ping, j.same_worker AS "same_worker?",
COALESCE(s.flow_status, s.workflow_as_code_status)::text AS "flow_status: Box<str>", r.ping AS last_ping, j.same_worker AS "same_worker?",
q.worker AS "worker?",
wp.ping_at AS "worker_last_ping?",
wp.memory_usage AS "worker_memory_usage?",
@@ -4898,11 +4989,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
.as_deref()
.and_then(|x| serde_json::from_str::<FlowStatus>(x).ok());
if !flow.same_worker.unwrap_or(false)
&& status.is_some_and(|s| {
s.modules
.get(0)
.is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. }))
})
&& status.as_ref().is_some_and(|s| s.is_not_yet_started())
{
let error_message = format!(
"Zombie flow detected: {} in workspace {}. It hasn't started yet, restarting it.",
@@ -4963,18 +5050,35 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
// worker name; this flow's recorded worker name still points at the
// dead process whose last ping can be under 60s old, and the memory
// signal is what lets us catch that window.
let memory_pct: Option<f64> = flow.worker_memory_total.and_then(|total| {
if total <= 0 {
return None;
}
let used = flow.worker_memory_usage.max(flow.worker_wm_memory_usage)?;
Some(used as f64 / total as f64)
});
let memory_pct: Option<f64> = zombie_worker_memory_pct(
flow.worker_memory_usage,
flow.worker_wm_memory_usage,
flow.worker_memory_total,
);
let oom_strong = memory_pct.is_some_and(|p| p >= 0.85);
let oom_moderate = memory_pct.is_some_and(|p| p >= 0.60);
let mem_pct_str = memory_pct
.map(|p| format!("{:.1}% of container limit", (p * 100.0).min(100.0)))
.unwrap_or_else(|| "memory unknown at last ping".to_string());
// When q.worker itself already shows OOM evidence the diagnosis below is
// already correct. Otherwise q.worker is likely a bystander (the outer
// queue-row worker) and the dropped transition was performed by a
// different worker on the same pod/group that OOM-died — go find it.
let q_worker_shows_oom = oom_moderate || worker_ping_stale == Some(true);
let culprit = if q_worker_shows_oom {
None
} else if let (Some(qw), Some(lp)) = (flow.worker.as_deref(), flow.last_ping) {
find_zombie_flow_culprit_worker(db, qw, lp).await
} else {
None
};
let culprit_pct = culprit.as_ref().and_then(|c| {
zombie_worker_memory_pct(c.memory_usage, c.wm_memory_usage, c.memory_total)
});
let culprit_pct_str =
culprit_pct.map(|p| format!("{:.1}% of its memory limit", (p * 100.0).min(100.0)));
let worker_info = if let Some(worker_name) = flow.worker.as_deref() {
let mut s = format!("\nWorker handling the flow: {worker_name}");
match (flow.worker_group.as_deref(), flow.worker_version.as_deref()) {
@@ -5005,8 +5109,11 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
(false, false, true) => format!(
"LIKELY OOM-KILLED — {mem_pct_str} at last ping (a replacement worker process may have started in the same pod under a new windmill worker name)"
),
(false, false, false) if culprit.is_some() => {
"still pinging with healthy memory — this is NOT the worker that performed the flow's final state transition (see likely culprit worker below)".to_string()
}
(false, false, false) => {
"worker still pinging with healthy memory — likely deadlocked or blocking on the state transition".to_string()
"worker still pinging with healthy memory — most likely a different worker performed and dropped the final transition (see hint); less likely: this worker deadlocked or is blocking on the state transition".to_string()
}
};
s.push_str(&format!("\nWorker last ping: {wp} ({age}s ago) — {status}"));
@@ -5051,20 +5158,88 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
.to_string()
};
let hint: String = match (worker_ping_stale, oom_moderate) {
(Some(_), true) => format!(
"\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)."
),
(Some(true), false) => {
"\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string()
}
(Some(false), false) => {
"\nWorker is still pinging and memory looked healthy at its last ping — most likely a deadlock or blocking call during the state transition. Capture a stack trace (e.g. via SIGQUIT) from the worker process. As a sanity check, also verify pod restart count in case a replacement worker process in the same pod has silently taken over.".to_string()
}
(None, _) => String::new(),
let culprit_info = if let Some(c) = culprit.as_ref() {
let age = (now - c.ping_at).num_seconds();
let rel = match c.ping_delta_secs {
Some(d) if d >= 0.0 => format!("{d:.0}s after"),
Some(d) => format!("{:.0}s before", -d),
None => "around".to_string(),
};
let loc =
if c.worker_instance.is_some() && c.worker_instance == flow.worker_instance {
format!(
"same pod/instance '{}'",
c.worker_instance.as_deref().unwrap()
)
} else if let Some(g) = c.worker_group.as_deref() {
format!("worker group '{g}'")
} else if let Some(inst) = c.worker_instance.as_deref() {
format!("instance '{inst}'")
} else {
"same pod/group".to_string()
};
let mem = match culprit_pct_str.as_deref() {
Some(p) => format!("was at {p}"),
None => "did not report memory".to_string(),
};
let mem_detail = match (c.memory_usage, c.wm_memory_usage, c.memory_total) {
(host, wm, Some(total)) => {
let used = host.max(wm);
match used {
Some(u) => format!(
" (memory at last ping: {} of {})",
fmt_mb(u),
fmt_mb(total)
),
None => format!(" (total available: {})", fmt_mb(total)),
}
}
_ => String::new(),
};
let q_name = flow.worker.as_deref().unwrap_or("the recorded worker");
format!(
"\nLikely culprit worker (on {loc}): {} — last pinged {} ({age}s ago, {rel} this flow's last ping) and then stopped pinging; it {mem} at that last ping{mem_detail}. This flow's dropped state transition was most likely performed by this worker and lost to its death (most likely OOM-kill), not a deadlock on {q_name}.",
c.worker, c.ping_at,
)
} else {
String::new()
};
let service_logs_info = match (flow.worker_instance.as_deref(), flow.worker_last_ping) {
let hint: String = if let Some(c) = culprit.as_ref() {
let q_name = flow.worker.as_deref().unwrap_or("the recorded worker");
match culprit_pct {
Some(p) if p >= 0.60 => format!(
"\nThis is almost certainly an OOM-kill on a *different* worker: {} (on the same pod/worker group) was at {} at its last ping right around this flow's transition, then stopped pinging. The flow's recorded worker ({q_name}) looks healthy because it is not the worker that performed the dropped transition. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`).",
c.worker,
culprit_pct_str.as_deref().unwrap_or("a high fraction of its limit"),
),
_ => format!(
"\nMost likely an OOM-kill on a *different* worker: {} (on the same pod/worker group) stopped pinging right around this flow's transition ({last_ping:?}); its memory may have spiked after its last ping or not been reported. The flow's recorded worker ({q_name}) looks healthy because it did not perform the dropped transition. First check that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Only if that worker was not OOM-killed, consider a deadlock on {q_name} and capture a stack trace (e.g. via SIGQUIT).",
c.worker,
),
}
} else {
match (worker_ping_stale, oom_moderate) {
(Some(_), true) => format!(
"\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)."
),
(Some(true), false) => {
"\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string()
}
(Some(false), false) => {
format!("\nThe flow's recorded worker is still pinging with healthy memory, but that worker is often NOT the one that performed the final state transition (in nested/subflow/forloop cases the last iteration runs on another worker). First check whether a different worker on the same pod / worker group was OOM-killed around {last_ping:?} (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`, and that pod's worker memory metrics). Only if no such worker died, treat this as a deadlock or blocking call on the recorded worker during the state transition and capture a stack trace (e.g. via SIGQUIT).")
}
(None, _) => String::new(),
}
};
// Pull logs for the worker (and around the time) we actually blame: the
// culprit's instance/last-ping when one was found, else q.worker's.
let (log_instance, log_ping) = match culprit.as_ref() {
Some(c) => (c.worker_instance.as_deref(), Some(c.ping_at)),
None => (flow.worker_instance.as_deref(), flow.worker_last_ping),
};
let service_logs_info = match (log_instance, log_ping) {
(Some(host), Some(wlp)) => {
let after = (wlp - chrono::Duration::seconds(90))
.to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
@@ -5118,13 +5293,25 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> {
};
let reason = format!(
"{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{hint}{service_logs_info}",
"{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{culprit_info}{hint}{service_logs_info}",
if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() {
format!("Flow was cancelled because subflow {id} ({base_url}/run/{id}?workspace={workspace_id})")
} else {
format!("Flow {id} ({base_url}/run/{id}?workspace={workspace_id}) was cancelled because it")
}
);
let reason = match between_steps_recovery_guidance(
db,
status.as_ref(),
id,
&workspace_id,
&base_url,
)
.await
{
Some(guidance) => format!("{reason}\n\n{guidance}"),
None => reason,
};
report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await;
cancel_zombie_flow_job(db, flow.id, &flow.workspace_id,
format!(r#"{reason}
@@ -5171,6 +5358,103 @@ Please check your worker logs for more details and feel free to report it to the
Ok(())
}
/// When a between-steps zombie's stuck step has every child recorded as a
/// `success` completion, the flow's state is fully derivable: only the final
/// state transition was lost to the worker failure, not any real work. In that
/// case return concrete restart-from-step recovery guidance to append to the
/// cancellation reason / critical alert. Returns `None` when the state isn't
/// derivable (some child missing or not successful), so the existing wording is
/// left untouched. Auto-recovery is deliberately not attempted (a re-driven
/// transition can OOM again on the same aggregated state; a human raises the
/// memory limit first, then restarts).
async fn between_steps_recovery_guidance(
db: &DB,
status: Option<&FlowStatus>,
flow_id: Uuid,
workspace_id: &str,
base_url: &str,
) -> Option<String> {
// The stuck module is the current step, left InProgress because the
// transition that would have marked it Success was dropped. It is only
// derivable when its own cursor reached the end (a serial fan-out reaped
// mid-iteration has unrun work left; while-loops are never derivable). Whether
// restart reuses the children or re-runs the step (final step, or one carrying a
// stop/skip/approval/sleep) is decided by the restart path against the flow
// definition, which the reaper doesn't load; the guidance states both outcomes
// rather than promising reuse the restart might decline.
let status = status?;
let idx = usize::try_from(status.step).ok()?;
let module = status.modules.get(idx)?;
if !module.is_between_steps_complete() {
return None;
}
let step_id = module.id();
// Only a top-level deployed flow exposes a working restart-from-step: the run page's
// "Re-start from" button is rendered only for job_kind == 'flow' (a flowpreview, even a
// pathful editor preview, or a singlestepflow does not qualify), and a subflow child
// restarts via its root. Match that surface exactly so the guidance never points at a
// button / endpoint that isn't there; leave the existing wording otherwise.
let restartable = sqlx::query_scalar!(
r#"SELECT (kind = 'flow' AND parent_job IS NULL) AS "restartable!"
FROM v2_job WHERE id = $1"#,
flow_id,
)
.fetch_one(db)
.await
.ok()?;
if !restartable {
return None;
}
// Children whose completion the lost transition would have aggregated: the
// loop/branchall iterations, or the single leaf/subflow child.
let child_ids: Vec<Uuid> = module
.flow_jobs()
.filter(|v| !v.is_empty())
.or_else(|| module.job().map(|j| vec![j]))?;
// Derivable only when every child is recorded as a success completion.
let success_children = sqlx::query_scalar!(
"SELECT count(*) FROM v2_job_completed
WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'",
workspace_id,
&child_ids,
)
.fetch_one(db)
.await
.ok()?
.unwrap_or(0);
if success_children != child_ids.len() as i64 {
return None;
}
let n = child_ids.len();
// For loop/branchall, name the completed iteration/branch count so the
// operator can confirm the whole fan-out is intact.
let iteration_hint = match module {
FlowStatusModule::InProgress { iterator: Some(_), .. } => {
format!(" (loop step, all {n} iterations completed)")
}
FlowStatusModule::InProgress { branchall: Some(_), .. } => {
format!(" (branchall step, all {n} branches completed)")
}
_ => String::new(),
};
Some(format!(
"RECOVERY: all {n} child job(s) of step `{step_id}`{iteration_hint} completed successfully; \
only the flow's final state transition was lost to the worker failure above (not any genuine failure), so \
the completed work is intact. To recover: first change the failure condition (raise the worker memory limit, \
e.g. k8s `resources.limits.memory`, or move the flow to a larger worker group), then restart from step \
`{step_id}`. Restart replays only the dropped transition and reuses the completed children where the step's \
result is fully derivable; a step that is the flow's last, or carries a stop/skip condition, an approval, or a \
sleep, is re-run instead (re-evaluating those on the larger worker).\n\
UI: open {base_url}/run/{flow_id}?workspace={workspace_id} and use \"Re-start from {step_id}\".\n\
API: POST {base_url}/api/w/{workspace_id}/jobs/restart/f/{flow_id} with body {{\"step_id\":\"{step_id}\"}}."
))
}
async fn cancel_zombie_flow_job(
db: &Pool<Postgres>,
id: Uuid,
@@ -5788,3 +6072,39 @@ mod strike_unarmed_tests {
assert_eq!(strike_unarmed(&mut seen, set(&["b"])), vec![key("b")]);
}
}
#[cfg(test)]
mod zombie_worker_memory_pct_tests {
use super::zombie_worker_memory_pct;
#[test]
fn takes_the_larger_of_the_two_readings() {
// Both present: the larger (higher-signal) reading wins, not either
// one unconditionally — a "simplify to `usage.or(wm_usage)`" refactor
// would silently under-report and miss OOMs.
let p = zombie_worker_memory_pct(Some(600), Some(900), Some(1000)).unwrap();
assert!((p - 0.9).abs() < f64::EPSILON);
}
#[test]
fn falls_back_to_whichever_reading_is_present() {
assert_eq!(
zombie_worker_memory_pct(Some(700), None, Some(1000)),
Some(0.7)
);
assert_eq!(
zombie_worker_memory_pct(None, Some(800), Some(1000)),
Some(0.8)
);
}
#[test]
fn none_when_no_usage_or_no_valid_total() {
assert_eq!(zombie_worker_memory_pct(None, None, Some(1000)), None);
assert_eq!(zombie_worker_memory_pct(Some(500), Some(500), None), None);
assert_eq!(
zombie_worker_memory_pct(Some(500), Some(500), Some(0)),
None
);
}
}
backend/tests/app_s3_onbehalf.rs
+192 -118
View File
@@ -1,7 +1,9 @@
//! Deployed-app S3 reads authorize on-behalf of the app author and are confined
//! to app provenance (declared keys or recent job outputs): a viewer cannot read
//! an arbitrary `file_key` as the author. Requires the `parquet` feature — the
//! real `apps_u/*` S3 handlers are gated on it.
//! Deployed-app S3 reads authorize on-behalf of the app author and are confined to
//! app provenance (declared keys or recent job outputs): an anonymous viewer cannot
//! read an arbitrary `file_key` as the author. A viewer on a full (unscoped) session
//! instead falls back to reading as THEMSELVES (bounded by their own S3 perms), so the
//! gate is exercised here through the anonymous identity it still fully protects.
//! Requires the `parquet` feature — the real `apps_u/*` S3 handlers are gated on it.
//!
//! `base` fixture: test-user (admin, SECRET_TOKEN); test-user-2 (non-admin,
//! SECRET_TOKEN_2, no S3 folder permission).
@@ -21,6 +23,19 @@ fn client() -> reqwest::Client {
reqwest::Client::new()
}
/// Mint an API token for test-user (admin) restricted to `scopes`.
async fn mint_scoped_token(port: u16, scopes: Vec<&str>) -> anyhow::Result<String> {
let resp = authed(
client().post(format!("http://localhost:{port}/api/users/tokens/create")),
ADMIN_TOKEN,
)
.json(&json!({ "label": "scoped", "scopes": scopes, "workspace_id": "test-workspace" }))
.send()
.await?;
assert_eq!(resp.status(), 201, "mint scoped token");
Ok(resp.text().await?)
}
fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder {
builder.header("Authorization", format!("Bearer {}", token))
}
@@ -50,62 +65,54 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool<Postgres>) -> anyhow:
.await?;
assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
// GET an app-scoped S3 route as `token`. No workspace storage is configured,
// so a request that clears the provenance gate fails later at the storage
// lookup (or the CE OSS stub), never with "File restricted" — which is what
// lets these assertions distinguish "gate passed" from "gate denied".
let get = |route: &str, token: &'static str| {
// GET an app-scoped S3 route ANONYMOUSLY. Anonymous callers have no viewer
// identity to fall back to, so the provenance gate still fully applies to them
// (unlike logged-in viewers, who now read as themselves — see the union test).
// No workspace storage is configured, so a request that clears the gate fails
// later at the storage lookup (or the CE OSS stub), never with the denial
// message — which is what lets these assertions distinguish pass from deny.
let get = |route: &str| {
let url = format!("{ws}/apps_u/{route}");
authed(client().get(url), token).send()
client().get(url).send()
};
let denied = |body: &str| body.contains("File restricted");
let denied = |body: &str| body.contains("is not accessible from this app");
// download_s3_file: author-on-behalf allowed for the declared key, denied for
// a key the app never declared (the confused-deputy guard).
let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}"), USER_TOKEN)
// download_s3_file: allowed for the declared key, denied for a key the app never
// declared (the confused-deputy guard).
let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}"))
.await?
.text()
.await?;
assert!(!denied(&body), "declared key must clear the gate: {body}");
let body = get(
&format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"),
USER_TOKEN,
)
.await?
.text()
.await?;
let body = get(&format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"))
.await?
.text()
.await?;
assert!(denied(&body), "non-provenance key must be denied: {body}");
// load_table_count and load_csv_preview enforce the same gate. The preview's
// numeric `limit`/`offset` must deserialize (regression: a flattened query
// struct 400s on them under serde_urlencoded).
let body = get(
&format!("load_table_count/{APP}?file_key={DECLARED}"),
USER_TOKEN,
)
.await?
.text()
.await?;
let body = get(&format!("load_table_count/{APP}?file_key={DECLARED}"))
.await?
.text()
.await?;
assert!(
!denied(&body),
"table_count declared key must clear the gate: {body}"
);
let body = get(
&format!("load_table_count/{APP}?file_key={NON_PROVENANCE}"),
USER_TOKEN,
)
.await?
.text()
.await?;
let body = get(&format!("load_table_count/{APP}?file_key={NON_PROVENANCE}"))
.await?
.text()
.await?;
assert!(
denied(&body),
"table_count non-provenance key must be denied: {body}"
);
let resp = get(
&format!("load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0"),
USER_TOKEN,
)
let resp = get(&format!(
"load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0"
))
.await?;
let status = resp.status();
let body = resp.text().await?;
@@ -116,23 +123,16 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool<Postgres>) -> anyhow:
);
// load_file_preview: `read_bytes_from` / `read_bytes_length` are required.
let resp = get(
&format!("load_file_preview/{APP}?file_key={DECLARED}"),
USER_TOKEN,
)
.await?;
let resp = get(&format!("load_file_preview/{APP}?file_key={DECLARED}")).await?;
assert_eq!(
resp.status(),
400,
"file_preview without byte range must 400: {}",
resp.text().await?
);
let body = get(
&format!(
"load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096"
),
USER_TOKEN,
)
let body = get(&format!(
"load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096"
))
.await?
.text()
.await?;
@@ -144,6 +144,94 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool<Postgres>) -> anyhow:
Ok(())
}
/// The viewer-perm union: a viewer on a full (unscoped) session is no longer hard-denied
/// by the provenance gate for a pre-existing file. It falls back to reading as ITSELF
/// (bounded by its own S3 perms downstream), while an anonymous caller (no identity) and
/// a scope-restricted token (can hit `apps_u/*` but not `job_helpers/*`, so the fallback
/// would be a new capability) both stay fully gated with the actionable denial.
#[sqlx::test(fixtures("base"))]
async fn test_deployed_app_s3_viewer_union(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let ws = format!("http://localhost:{port}/api/w/test-workspace");
let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN)
.json(&json!({
"path": APP,
"summary": "s3 viewer union test",
"value": {},
"policy": {
"execution_mode": "anonymous",
"triggerables": {},
"allowed_s3_keys": [{ "s3_path": DECLARED }]
}
}))
.send()
.await?;
assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
let url = format!("{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}");
// Anonymous: still gated. The denial is the actionable message and echoes the key.
let body = client().get(&url).send().await?.text().await?;
assert!(
body.contains("is not accessible from this app"),
"anonymous viewer must stay gated with the actionable denial: {body}"
);
assert!(
body.contains(NON_PROVENANCE),
"denial must echo the requested key: {body}"
);
// Logged-in viewer: no longer hard-denied — the gate delegates to reading as the
// viewer, so the request falls through to the storage read (no gate denial in
// EITHER the old or new form). No workspace storage is configured here, so it
// surfaces a downstream storage/OSS error, not a gate denial.
let body = authed(client().get(&url), USER_TOKEN)
.send()
.await?
.text()
.await?;
assert!(
!body.contains("is not accessible from this app") && !body.contains("File restricted"),
"logged-in viewer must delegate to its own read, not be gate-denied: {body}"
);
// Scope-restricted token: an `apps:read:<app>` token reaches this route but is
// REJECTED by the route-scope middleware on `job_helpers/*`, so it must NOT get the
// viewer fallback (that would be a capability it cannot obtain directly). It stays
// gated with the denial, unlike the unscoped session above.
let apps_read_scope = format!("apps:read:{APP}");
let scoped = mint_scoped_token(port, vec![apps_read_scope.as_str()]).await?;
let body = authed(client().get(&url), &scoped)
.send()
.await?
.text()
.await?;
assert!(
body.contains("is not accessible from this app"),
"scope-restricted token must stay gated, not get the viewer fallback: {body}"
);
// A filter-tags-only token carries no real scope restriction (the route-scope
// middleware treats it as unscoped), so it can read via job_helpers directly and
// MUST get the viewer fallback here — not be gated like a genuinely scoped token.
let tag_only = mint_scoped_token(port, vec!["if_jobs:filter_tags:default"]).await?;
let body = authed(client().get(&url), &tag_only)
.send()
.await?
.text()
.await?;
assert!(
!body.contains("is not accessible from this app") && !body.contains("File restricted"),
"filter-tags-only token is effectively unscoped and must delegate, not be gated: {body}"
);
Ok(())
}
/// Mint a presigned bearer (`exp=..&sig=..`) exactly as `sign_s3_objects` does:
/// `HMAC-SHA256(workspace_key, "file_key={s3}&exp={exp}")` (no storage param, since
/// these routes send none). `validate_s3_signature` is `private`-gated, so this test
@@ -198,16 +286,19 @@ async fn test_deployed_app_s3_presigned_bypasses_gate(db: Pool<Postgres>) -> any
let url = format!("{ws}/apps_u/{route}");
authed(client().get(url), token).send()
};
let denied = |body: &str| body.contains("File restricted");
let denied = |body: &str| body.contains("is not accessible from this app");
// Control: NON_PROVENANCE without a signature is denied by the gate.
let body = get(
format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"),
USER_TOKEN,
)
.await?
.text()
.await?;
// Control: NON_PROVENANCE without a signature is denied by the gate. Sent
// anonymously — a logged-in viewer would instead fall back to reading as
// themselves, so anonymous is the identity that isolates the presigned bypass.
let body = client()
.get(format!(
"{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}"
))
.send()
.await?
.text()
.await?;
assert!(
denied(&body),
"unsigned non-provenance key must be denied: {body}"
@@ -304,12 +395,14 @@ async fn seed_completed_job(
}
/// A deployed app that renders S3 files it produced (e.g. a SQL query persisted to
/// S3 by a component) must clear the provenance gate for the viewer whose own app
/// run produced them, while (a) a viewer cannot forge provenance by running a
/// runnable directly (no app marker), (b) another app's outputs stay denied, and
/// (c) another viewer's outputs stay denied (cross-viewer isolation). Provenance is
/// keyed on the app-origination marker (`trigger_kind='app'` + `trigger=<app path>`)
/// that `execute_component` stamps, plus `created_by = <this caller>` for isolation.
/// S3 by a component) must clear the provenance gate for the caller whose own app run
/// produced them, while (a) provenance cannot be forged by running a runnable directly
/// (no app marker), (b) another app's outputs stay denied, and (c) another caller's
/// outputs stay denied (per-caller isolation). Provenance is keyed on the
/// app-origination marker (`trigger_kind='app'` + `trigger=<app path>`) that
/// `execute_component` stamps, plus `created_by = <this caller>` for isolation.
/// Exercised anonymously: the gate still fully governs anonymous callers, whereas a
/// logged-in viewer would instead fall back to reading as themselves.
#[sqlx::test(fixtures("base"))]
async fn test_deployed_app_s3_onbehalf_flow_script_provenance(
db: Pool<Postgres>,
@@ -322,10 +415,10 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance(
const FS_APP: &str = "u/test-user/s3flowscript";
const OTHER_APP: &str = "u/test-user/other_app";
// Produced by test-user-2's own app run of THIS app.
const USER_KEY: &str = "results/user2_output.parquet";
// Produced by test-user's own app run of THIS app.
const ADMIN_KEY: &str = "results/admin_output.parquet";
// Produced by the anonymous caller's own app run of THIS app.
const OWN_KEY: &str = "results/own_output.parquet";
// Produced by a DIFFERENT caller's app run of THIS app → isolation, must stay denied.
const OTHER_CALLER_KEY: &str = "results/user2_output.parquet";
// Produced by an app run of a DIFFERENT app → must stay denied.
const OTHER_APP_KEY: &str = "results/other_app_output.parquet";
// Produced by a DIRECT run (no app marker) → the forgery attempt, must stay denied.
@@ -342,64 +435,49 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance(
.await?;
assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?);
// Seed the produced-file jobs (all within the 3h window).
seed_completed_job(&db, "test-user-2", Some(FS_APP), USER_KEY).await?;
seed_completed_job(&db, "test-user", Some(FS_APP), ADMIN_KEY).await?;
seed_completed_job(&db, "test-user-2", Some(OTHER_APP), OTHER_APP_KEY).await?;
seed_completed_job(&db, "test-user-2", None, FORGED_KEY).await?;
// Seed the produced-file jobs (all within the 3h window). The gate's `created_by`
// filter uses "anonymous" for an unauthenticated caller.
seed_completed_job(&db, "anonymous", Some(FS_APP), OWN_KEY).await?;
seed_completed_job(&db, "test-user-2", Some(FS_APP), OTHER_CALLER_KEY).await?;
seed_completed_job(&db, "anonymous", Some(OTHER_APP), OTHER_APP_KEY).await?;
seed_completed_job(&db, "anonymous", None, FORGED_KEY).await?;
let get = |route: &str, token: &'static str| {
let denied = |body: &str| body.contains("is not accessible from this app");
// Anonymous GET (borrows `ws`, reusable across calls: the URL is built before the
// `async move` so only the owned `url` is moved into the future, not `ws`).
let anon_body = |route: String| {
let url = format!("{ws}/apps_u/{route}");
authed(client().get(url), token).send()
};
let denied = |body: &str| body.contains("File restricted");
let body_of = |route: String, token: &'static str| async move {
get(&route, token).await.unwrap().text().await.unwrap()
async move {
client()
.get(url)
.send()
.await
.unwrap()
.text()
.await
.unwrap()
}
};
// The viewer's own app run's output clears the gate (the case that regressed to
// "File restricted").
let body = body_of(
format!("download_s3_file/{FS_APP}?s3={USER_KEY}"),
USER_TOKEN,
)
.await;
// The caller's own app run's output clears the gate (the case that regressed to
// a hard denial).
let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OWN_KEY}")).await;
assert!(
!denied(&body),
"viewer's own app-produced key must clear the gate: {body}"
"caller's own app-produced key must clear the gate: {body}"
);
// The admin viewer's own app run's output clears — the gate has no admin bypass,
// it just matches the caller's own runs.
let body = body_of(
format!("download_s3_file/{FS_APP}?s3={ADMIN_KEY}"),
ADMIN_TOKEN,
)
.await;
assert!(
!denied(&body),
"admin's own app-produced key must clear the gate: {body}"
);
// Cross-viewer isolation: the admin cannot pull test-user-2's result even though
// it is a genuine app-marked job of the same app (no admin bypass either).
let body = body_of(
format!("download_s3_file/{FS_APP}?s3={USER_KEY}"),
ADMIN_TOKEN,
)
.await;
// Per-caller isolation: another caller's result stays denied even though it is a
// genuine app-marked job of the same app.
let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_CALLER_KEY}")).await;
assert!(
denied(&body),
"another viewer's app-produced key must stay denied (isolation): {body}"
"another caller's app-produced key must stay denied (isolation): {body}"
);
// A key produced by a direct run (no app marker) stays denied — the forgery the
// app-origination marker closes.
let body = body_of(
format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}"),
USER_TOKEN,
)
.await;
let body = anon_body(format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}")).await;
assert!(
denied(&body),
"key from a direct run (no app marker) must stay denied: {body}"
@@ -407,11 +485,7 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance(
// A key produced by a DIFFERENT app stays denied — provenance is scoped to THIS
// app's path.
let body = body_of(
format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}"),
USER_TOKEN,
)
.await;
let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}")).await;
assert!(
denied(&body),
"key produced by a different app must stay denied: {body}"
backend/tests/nativets_jobs.rs
+37
View File
@@ -263,6 +263,43 @@ export function main(): number[] {
);
}
// -- result + wm_labels carrying a NUL: must complete, jsonb-safe & text[]-safe --
{
// `\u0000` here is 6 literal chars in the raw string; the JS runtime emits
// a real U+0000. It aborts the jsonb result insert (22P05) and, via
// wm_labels, the `text[]` labels update - both in the completion tx.
// `nul`/label come back stripped; `literal` (escaped backslash + text
// "u0000", no real NUL) survives untouched.
let result = push_and_wait(
&db,
RunJob::from(nativets_code(
r#"//native
export function main(): {nul: string, literal: string, wm_labels: string[]} {
return { nul: "a\u0000b", literal: "a\\u0000b", wm_labels: ["x\u0000y"] };
}
"#,
)),
&mut listener,
)
.await;
assert!(result.success, "nul_result failed: {:?}", result.result);
let val = result.json_result().unwrap();
assert_eq!(val["nul"], serde_json::json!("ab"));
assert_eq!(val["literal"], serde_json::json!("a\\u0000b"));
// The wm_labels entry is persisted to the `text[]` column NUL-free.
let labels: Option<Vec<String>> =
sqlx::query_scalar("SELECT labels FROM v2_job WHERE id = $1")
.bind(result.id)
.fetch_one(&db)
.await
.unwrap();
let labels = labels.unwrap_or_default();
assert!(labels.iter().any(|l| l == "xy"), "expected stripped label, got {labels:?}");
assert!(!labels.iter().any(|l| l.contains('\0')), "labels must be NUL-free: {labels:?}");
}
killpill.send();
Ok(())
}
backend/tests/suspend_resume.rs
+202
View File
@@ -352,6 +352,208 @@ mod suspend_resume {
Ok(())
}
/// The UI "Resume" button (POST /jobs_u/flow/resume_suspended/:job_id) must reject the
/// triggerer approving their own self_approval_disabled step even when they own the flow
/// path: only admins are exempt from the self-approval restriction, so owning the runnable
/// does not grant the right to self-approve.
#[cfg(feature = "enterprise")]
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_self_approval_disabled_blocks_ui_resume_for_owner(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let flow_with_self_approval_disabled: FlowValue = serde_json::from_value(json!({
"modules": [{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
"required_events": 1,
"user_auth_required": true,
"self_approval_disabled": true
}
}, {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
}))
.unwrap();
// Push as a non-admin who owns the flow path, so the owner shortcut is exercised.
let flow = RunJob::from(JobPayload::RawFlow {
value: flow_with_self_approval_disabled,
path: Some("u/test-user-2/test_ui_resume".to_string()),
restarted_from: None,
})
.push_as(&db, "test-user-2", "test2@windmill.dev")
.await;
let queue = listen_for_queue(&db).await;
let db_ = db.clone();
in_test_worker(
&db,
async move {
let db = db_;
wait_until_flow_suspends(flow, queue, &db).await;
let token = windmill_common::auth::create_token_for_owner(
&db,
"test-workspace",
"u/test-user-2",
"test-token",
100,
"test2@windmill.dev",
&Uuid::nil(),
None,
None,
)
.await
.unwrap();
// Resume via the UI endpoint as the owner who triggered the flow.
let response = reqwest::Client::new()
.post(format!(
"http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}"
))
.header("Authorization", format!("Bearer {token}"))
.header("Content-Type", "application/json")
.body("{}")
.send()
.await
.unwrap();
let status = response.status();
assert!(
status == reqwest::StatusCode::FORBIDDEN,
"Self-approval via the UI resume endpoint should be blocked for the owner when \
self_approval_disabled=true. Expected 403 Forbidden, got {}. Response: {}",
status,
response.text().await.unwrap_or_default()
);
},
port,
)
.await;
server.close().await.unwrap();
Ok(())
}
/// self_approval_disabled must hold even when user_auth_required is not set: the worker must
/// persist the condition and the resume boundary must enforce it for the authenticated
/// triggerer. The triggerer here is a non-owner (folder path they don't own), so the owner
/// shortcut is not involved and this exercises the persistence + authenticated-check path.
#[cfg(feature = "enterprise")]
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_self_approval_disabled_without_user_auth_required(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// self_approval_disabled without user_auth_required (as a raw-flow/CLI author could set).
let flow_value: FlowValue = serde_json::from_value(json!({
"modules": [{
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
"required_events": 1,
"self_approval_disabled": true
}
}, {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
}))
.unwrap();
// Folder path test-user-2 does not own -> non-owner triggerer (no folders in the base
// fixture), so the owner shortcut is bypassed and only persistence matters here.
let flow = RunJob::from(JobPayload::RawFlow {
value: flow_value,
path: Some("f/system/test_persist".to_string()),
restarted_from: None,
})
.push_as(&db, "test-user-2", "test2@windmill.dev")
.await;
let queue = listen_for_queue(&db).await;
let db_ = db.clone();
in_test_worker(
&db,
async move {
let db = db_;
wait_until_flow_suspends(flow, queue, &db).await;
let token = windmill_common::auth::create_token_for_owner(
&db,
"test-workspace",
"u/test-user-2",
"test-token",
100,
"test2@windmill.dev",
&Uuid::nil(),
None,
None,
)
.await
.unwrap();
let response = reqwest::Client::new()
.post(format!(
"http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}"
))
.header("Authorization", format!("Bearer {token}"))
.header("Content-Type", "application/json")
.body("{}")
.send()
.await
.unwrap();
let status = response.status();
assert!(
status == reqwest::StatusCode::FORBIDDEN,
"Self-approval should be blocked when self_approval_disabled=true even without \
user_auth_required. Expected 403 Forbidden, got {}. Response: {}",
status,
response.text().await.unwrap_or_default()
);
},
port,
)
.await;
server.close().await.unwrap();
Ok(())
}
/// Test that self-approval WORKS when self_approval_disabled is false (default behavior).
///
/// This is the complementary test to test_self_approval_disabled_blocks_owner_resume.
backend/tests/zombie_flow_recovery.rs
+661
View File
@@ -0,0 +1,661 @@
//! Regression test for hand-recovery of between-steps zombie flows.
//!
//! When a worker is OOM-killed mid state-transition, the zombie monitor
//! (`handle_zombie_flows` → `cancel_job` with force) reaps the flow: it lands in
//! `v2_job_completed` as `canceled`, with its `flow_status` preserved: the step
//! whose transition was lost stays `InProgress` even though all its children
//! completed successfully. This test reproduces that exact terminal state and
//! asserts that a hand-restart from the stuck step reuses every completed child
//! (no re-run) and the flow reaches success.
//!
//! The reaper itself lives in the `windmill` binary crate and is unreachable
//! from an integration test, so we reproduce the state `cancel_job(force)`
//! leaves behind directly; the fix under test is the restart-resolution path,
//! not the detection query.
#![cfg(feature = "deno_core")]
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_common::flow_status::{BranchChosen, FlowStatus, RestartedFrom};
use windmill_common::flows::FlowValue;
use windmill_common::jobs::JobPayload;
use windmill_test_utils::*;
/// Child job UUID for a top-level step in a completed flow's `flow_status`
/// (optionally the iteration index for a ForLoop / BranchAll container).
async fn child_job_id_for_step(
db: &Pool<Postgres>,
flow_job_id: uuid::Uuid,
step_id: &str,
iter: Option<usize>,
) -> uuid::Uuid {
let raw: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
flow_job_id
)
.fetch_one(db)
.await
.unwrap()
.expect("flow_status missing");
let status: FlowStatus = serde_json::from_value(raw).expect("parse flow_status");
let module = status
.modules
.iter()
.find(|m| m.id() == step_id)
.expect("step in flow_status");
match iter {
Some(i) => module.flow_jobs().expect("flow_jobs")[i],
None => module.job().expect("job"),
}
}
/// A between-steps zombie whose fan-out completed but whose final transition was
/// lost can be hand-restarted from the stuck step, reusing every completed child
/// (including the last iteration) and reaching success.
#[sqlx::test(fixtures("base", "hello"))]
async fn test_between_steps_zombie_restart_reuses_all_children(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// A fan-out ForLoop `fanout` (2 iterations) followed by `after`, which
// consumes the loop's aggregated result. In the zombie scenario `fanout`
// finished all iterations but its final transition was lost, so `after`
// never ran.
let flow_value: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "fanout",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "['a', 'b']" },
"skip_failures": false,
"parallel": false,
"modules": [{
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
"content": "export function main(v: string) { return v }"
}
}]
}
},
{
"id": "after",
"value": {
"type": "rawscript",
"language": "deno",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
"content": "export function main(loop_res: string[]) { return loop_res.join(',') }"
}
}
]
}))
.unwrap();
// Run to completion to obtain real, successful child jobs.
let full_run = RunJob::from(JobPayload::RawFlow {
value: flow_value.clone(),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, port)
.await;
assert!(full_run.success, "baseline run should succeed");
assert_eq!(full_run.json_result().unwrap(), json!("a,b"));
let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await;
let orig_iter1 = child_job_id_for_step(&db, full_run.id, "fanout", Some(1)).await;
let orig_after = child_job_id_for_step(&db, full_run.id, "after", None).await;
// Reproduce the zombie-reaper's terminal state: cancelled by `monitor` with
// `flow_status` frozen mid-transition: `fanout` still `InProgress` (all
// iterations done), `after` never reached.
let mut flow_status: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
full_run.id
)
.fetch_one(&db)
.await?
.expect("flow_status");
for m in flow_status["modules"].as_array_mut().unwrap() {
match m["id"].as_str() {
Some("fanout") => {
m["type"] = json!("InProgress");
// A reaped loop keeps its cursor at the last iteration.
m["iterator"] = json!({ "index": 1, "itered_len": 2 });
}
Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }),
_ => {}
}
}
flow_status["step"] = json!(0);
sqlx::query!(
"UPDATE v2_job_completed
SET status = 'canceled', canceled_by = 'monitor', canceled_reason = 'zombie flow',
flow_status = $2
WHERE id = $1",
full_run.id,
flow_status,
)
.execute(&db)
.await?;
// Hand-restart from the stuck step. `fanout` is recognised as a derivable
// between-steps zombie (all children succeeded), so it is reused verbatim and
// only the dropped transition onward is replayed. `Some(0)` is the exact value the
// run page's "Re-start from" button sends (a whole-step restart), not `None`.
let restarted = RunJob::from(JobPayload::RestartedFlow {
completed_job_id: full_run.id,
step_id: "fanout".into(),
branch_or_iteration_n: Some(0),
flow_version: None,
branch_chosen: None,
nested: None,
})
.run_until_complete(&db, false, port)
.await;
// Flow reaches success, reusing the loop's aggregated result.
assert!(
restarted.success,
"restarted zombie flow should succeed: {:?}",
restarted.json_result()
);
assert_eq!(restarted.json_result().unwrap(), json!("a,b"));
// Every completed loop iteration reuses its original child job (no re-run);
// only `after`, which never ran, executes fresh.
let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await;
let new_iter1 = child_job_id_for_step(&db, restarted.id, "fanout", Some(1)).await;
let new_after = child_job_id_for_step(&db, restarted.id, "after", None).await;
assert_eq!(new_iter0, orig_iter0, "loop iteration 0 must be reused");
assert_eq!(new_iter1, orig_iter1, "loop iteration 1 must be reused");
assert_ne!(new_after, orig_after, "`after` should run fresh");
Ok(())
}
/// A serial for-loop reaped *between* iterations (an all-success prefix, but the
/// cursor not yet at the last iteration) must NOT be treated as complete: reuse
/// would silently drop the remaining iterations. A downstream `after` step makes
/// the loop non-final, so the ONLY thing that can prevent reuse here is the
/// cursor-completeness guard; if it regresses, `after` would consume a truncated
/// loop result and this test fails. Restart must re-run the whole loop instead.
#[sqlx::test(fixtures("base", "hello"))]
async fn test_mid_iteration_zombie_not_reused(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let flow_value: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "fanout",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "['a', 'b', 'c']" },
"skip_failures": false,
"parallel": false,
"modules": [{
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
"content": "export function main(v: string) { return v }"
}
}]
}
},
{
"id": "after",
"value": {
"type": "rawscript",
"language": "deno",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
"content": "export function main(loop_res: string[]) { return loop_res.join(',') }"
}
}
]
}))
.unwrap();
let full_run = RunJob::from(JobPayload::RawFlow {
value: flow_value.clone(),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, port)
.await;
assert!(full_run.success);
let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await;
// Reap after iteration 0: the loop is InProgress with the cursor still on
// iteration 0 (of 3), only iteration 0 recorded; `after` never reached.
let mut flow_status: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
full_run.id
)
.fetch_one(&db)
.await?
.expect("flow_status");
for m in flow_status["modules"].as_array_mut().unwrap() {
match m["id"].as_str() {
Some("fanout") => {
m["type"] = json!("InProgress");
m["iterator"] = json!({ "index": 0, "itered_len": 3 });
m["flow_jobs"] = json!([m["flow_jobs"][0]]);
m["flow_jobs_success"] = json!([true]);
}
Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }),
_ => {}
}
}
flow_status["step"] = json!(0);
sqlx::query!(
"UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor',
flow_status = $2 WHERE id = $1",
full_run.id,
flow_status,
)
.execute(&db)
.await?;
let restarted = RunJob::from(JobPayload::RestartedFlow {
completed_job_id: full_run.id,
step_id: "fanout".into(),
branch_or_iteration_n: None,
flow_version: None,
branch_chosen: None,
nested: None,
})
.run_until_complete(&db, false, port)
.await;
// The loop re-runs from scratch: all three iterations execute (so `after` sees
// "a,b,c", not a truncated "a"), and iteration 0 is a fresh job.
assert!(
restarted.success,
"restart should re-run the loop and succeed: {:?}",
restarted.json_result()
);
assert_eq!(restarted.json_result().unwrap(), json!("a,b,c"));
let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await;
assert_ne!(
new_iter0, orig_iter0,
"iteration 0 must re-run, not be reused"
);
Ok(())
}
/// A nested restart request targets an inner step of the restart-step container.
/// Even when that container is an eligible between-steps zombie, reuse must NOT
/// fire (it would skip the whole container and ignore the explicit nested target).
/// The inner step must re-run.
#[sqlx::test(fixtures("base", "hello"))]
async fn test_nested_restart_not_swallowed_by_zombie_reuse(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
// `branch` is a BranchOne (single child, so branch_or_iteration_n is None on
// restart: the exact shape that would trip zombie reuse) with two inner steps,
// followed by a downstream `after`.
let flow_value: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "branch",
"value": {
"type": "branchone",
"default": [],
"branches": [{
"expr": "true",
"modules": [
{
"id": "inner_first",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {},
"content": "export function main() { return 'first' }"
}
},
{
"id": "inner_second",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"first": { "type": "javascript", "expr": "results.inner_first" }
},
"content": "export function main(first: string) { return `${first}|second` }"
}
}
]
}]
}
},
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"b": { "type": "javascript", "expr": "results.branch" }
},
"content": "export function main(b: string) { return `after:${b}` }"
}
}
]
}))
.unwrap();
let full_run = RunJob::from(JobPayload::RawFlow {
value: flow_value.clone(),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, port)
.await;
assert!(full_run.success);
assert_eq!(full_run.json_result().unwrap(), json!("after:first|second"));
let branch_child = child_job_id_for_step(&db, full_run.id, "branch", None).await;
let orig_inner_second = child_job_id_for_step(&db, branch_child, "inner_second", None).await;
// Reap `branch` as a between-steps zombie (its child completed, transition lost);
// `after` never reached.
let mut flow_status: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
full_run.id
)
.fetch_one(&db)
.await?
.expect("flow_status");
for m in flow_status["modules"].as_array_mut().unwrap() {
match m["id"].as_str() {
Some("branch") => m["type"] = json!("InProgress"),
Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }),
_ => {}
}
}
flow_status["step"] = json!(0);
sqlx::query!(
"UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor',
flow_status = $2 WHERE id = $1",
full_run.id,
flow_status,
)
.execute(&db)
.await?;
// Nested restart: re-run `inner_second` inside `branch`. Zombie reuse must step
// aside so the nested chain is honored.
let restarted = RunJob::from(JobPayload::RestartedFlow {
completed_job_id: full_run.id,
step_id: "branch".into(),
branch_or_iteration_n: None,
flow_version: None,
branch_chosen: Some(BranchChosen::Branch { branch: 0 }),
nested: Some(Box::new(RestartedFrom {
flow_job_id: branch_child,
step_id: "inner_second".into(),
branch_or_iteration_n: None,
flow_version: None,
branch_chosen: None,
nested: None,
})),
})
.run_until_complete(&db, false, port)
.await;
assert!(
restarted.success,
"nested restart of a zombie container should succeed: {:?}",
restarted.json_result()
);
assert_eq!(
restarted.json_result().unwrap(),
json!("after:first|second")
);
let new_branch_child = child_job_id_for_step(&db, restarted.id, "branch", None).await;
let new_inner_second = child_job_id_for_step(&db, new_branch_child, "inner_second", None).await;
assert_ne!(
new_inner_second, orig_inner_second,
"the nested target inner_second must re-run, not be skipped by zombie reuse"
);
Ok(())
}
/// A raw-flow (editor preview) restart queues the request's CURRENT definition, which the editor
/// allows to differ from the completed run. Zombie reuse must not fire there: it would validate the
/// stored step and synthesize Success from the old children, skipping the user's edit. The edited
/// step must re-run and downstream must observe its new result.
#[sqlx::test(fixtures("base", "hello"))]
async fn test_raw_flow_restart_does_not_reuse_edited_step(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let flow_of = |suffix: &str| -> FlowValue {
serde_json::from_value(json!({
"modules": [
{
"id": "fanout",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "['a', 'b']" },
"skip_failures": false,
"parallel": false,
"modules": [{
"id": "inner",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
"content": format!("export function main(v: string) {{ return v + '{suffix}' }}")
}
}]
}
},
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
"content": "export function main(loop_res: string[]) { return loop_res.join(',') }"
}
}
]
}))
.unwrap()
};
let full_run =
RunJob::from(JobPayload::RawFlow { value: flow_of(""), path: None, restarted_from: None })
.run_until_complete(&db, false, port)
.await;
assert!(full_run.success);
assert_eq!(full_run.json_result().unwrap(), json!("a,b"));
let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await;
let mut flow_status: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
full_run.id
)
.fetch_one(&db)
.await?
.expect("flow_status");
for m in flow_status["modules"].as_array_mut().unwrap() {
match m["id"].as_str() {
Some("fanout") => {
m["type"] = json!("InProgress");
m["iterator"] = json!({ "index": 1, "itered_len": 2 });
}
Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }),
_ => {}
}
}
flow_status["step"] = json!(0);
sqlx::query!(
"UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor',
flow_status = $2 WHERE id = $1",
full_run.id,
flow_status,
)
.execute(&db)
.await?;
let restarted = RunJob::from(JobPayload::RawFlow {
value: flow_of("X"),
path: None,
restarted_from: Some(RestartedFrom {
flow_job_id: full_run.id,
step_id: "fanout".into(),
branch_or_iteration_n: None,
flow_version: None,
branch_chosen: None,
nested: None,
}),
})
.run_until_complete(&db, false, port)
.await;
// The edited step must run: results reflect the new definition, not the reused old children.
assert!(
restarted.success,
"edited raw-flow restart should succeed: {:?}",
restarted.json_result()
);
assert_eq!(restarted.json_result().unwrap(), json!("aX,bX"));
let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await;
assert_ne!(
new_iter0, orig_iter0,
"the edited fanout step must re-run, not be reused"
);
Ok(())
}
/// Only a flow reaped by the zombie monitor (canceled_by = 'monitor') is eligible for reuse. A
/// plain force-cancel at the same boundary (a child succeeded, its parent transition not yet
/// landed) yields the identical InProgress/all-success shape but must keep restart-from-step
/// semantics: the selected step re-runs.
#[sqlx::test(fixtures("base", "hello"))]
async fn test_non_monitor_cancel_is_not_reused(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let flow_value: FlowValue = serde_json::from_value(json!({
"modules": [
{
"id": "fanout",
"value": {
"type": "forloopflow",
"iterator": { "type": "javascript", "expr": "['a', 'b']" },
"skip_failures": false,
"parallel": false,
"modules": [{
"id": "inner",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
"content": "export function main(v: string) { return v }"
}
}]
}
},
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
"content": "export function main(loop_res: string[]) { return loop_res.join(',') }"
}
}
]
}))
.unwrap();
let full_run = RunJob::from(JobPayload::RawFlow {
value: flow_value.clone(),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, port)
.await;
assert!(full_run.success);
let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await;
// Same frozen-transition shape as a zombie, but canceled by a USER, not the monitor.
let mut flow_status: serde_json::Value = sqlx::query_scalar!(
"SELECT flow_status FROM v2_job_completed WHERE id = $1",
full_run.id
)
.fetch_one(&db)
.await?
.expect("flow_status");
for m in flow_status["modules"].as_array_mut().unwrap() {
match m["id"].as_str() {
Some("fanout") => {
m["type"] = json!("InProgress");
m["iterator"] = json!({ "index": 1, "itered_len": 2 });
}
Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }),
_ => {}
}
}
flow_status["step"] = json!(0);
sqlx::query!(
"UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'admin',
flow_status = $2 WHERE id = $1",
full_run.id,
flow_status,
)
.execute(&db)
.await?;
let restarted = RunJob::from(JobPayload::RestartedFlow {
completed_job_id: full_run.id,
step_id: "fanout".into(),
branch_or_iteration_n: None,
flow_version: None,
branch_chosen: None,
nested: None,
})
.run_until_complete(&db, false, port)
.await;
assert!(restarted.success);
let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await;
assert_ne!(
new_iter0, orig_iter0,
"a non-monitor cancel must re-run the step, not reuse the child"
);
Ok(())
}
backend/windmill-ai/src/utils.rs
+69 -9
View File
@@ -18,20 +18,19 @@ lazy_static::lazy_static! {
/// use it instead of the shared `HTTP_CLIENT`. Redirects are governed by
/// `ALLOW_AI_BASE_URL_REDIRECTS` (disabled by default). Mirrors the API proxy
/// client (windmill-api/src/ai.rs).
///
/// This pooled client does no DNS pinning: callers reaching a user-controlled
/// base_url must go through [`pinned_ai_client_for`] so the connect targets
/// the SSRF-validated address (DNS-rebinding TOCTOU). It is the safe default
/// only for trusted/fixed hosts.
pub static ref AI_HTTP_CLIENT: reqwest::Client = {
let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS {
if *ALLOW_AI_BASE_URL_REDIRECTS {
tracing::warn!(
"ALLOW_AI_BASE_URL_REDIRECTS is enabled - the AI HTTP client will follow \
redirects, weakening SSRF protection on provider base URLs"
);
reqwest::redirect::Policy::default()
} else {
reqwest::redirect::Policy::none()
};
configure_client(reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.connect_timeout(std::time::Duration::from_secs(10))
.redirect(redirect))
}
ai_http_client_builder()
.build()
.expect("Failed to build AI HTTP client - check system TLS configuration")
};
@@ -64,6 +63,67 @@ lazy_static::lazy_static! {
};
}
/// Shared configuration for every client that targets a user-configured AI
/// provider `base_url` (the pooled [`AI_HTTP_CLIENT`] and per-request DNS-pinned
/// clients from [`pinned_ai_client_for`]). Redirects are disabled by default
/// because the SSRF check on base_url is single-shot; DNS pinning likewise only
/// covers the original host, so a redirect could bounce a validated public host
/// into a private/internal one (see `ALLOW_AI_BASE_URL_REDIRECTS`).
pub fn ai_http_client_builder() -> reqwest::ClientBuilder {
let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS {
reqwest::redirect::Policy::default()
} else {
reqwest::redirect::Policy::none()
};
configure_client(
reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.connect_timeout(std::time::Duration::from_secs(10))
.redirect(redirect),
)
}
/// Build the client for a single outbound AI request to `url`, pinning DNS to
/// the SSRF-validated address so the connect cannot rebind to an internal IP
/// after the check (DNS-rebinding TOCTOU).
///
/// Returns the shared pooled [`AI_HTTP_CLIENT`] unchanged when there is nothing
/// to pin — an IP-literal host, or a deployment that opted into private AI
/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint
/// as `AIProvider::get_base_url` apply, so this is consistent with the
/// credential-time validation while additionally closing the connect-time window.
pub async fn pinned_ai_client_for(
url: &str,
) -> windmill_common::error::Result<std::borrow::Cow<'static, reqwest::Client>> {
use std::borrow::Cow;
use windmill_common::error::{to_anyhow, Error};
use windmill_common::ssrf::SsrfValidationError;
if *crate::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS {
return Ok(Cow::Borrowed(&AI_HTTP_CLIENT));
}
let target = windmill_common::ssrf::validate_url_for_ssrf(url)
.await
.map_err(|e| match e {
e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!(
"{e}. If you need to use private/internal AI endpoints, \
set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable"
)),
e => Error::from(e),
})?;
if target.pinned_addrs().is_empty() {
return Ok(Cow::Borrowed(&AI_HTTP_CLIENT));
}
let client = target
.apply_dns_pinning(ai_http_client_builder())
.build()
.map_err(to_anyhow)?;
Ok(Cow::Owned(client))
}
/// AWS Bedrock do not handle structured output query param, so we use a tool for structured output. Same for every Claude models.
pub fn should_use_structured_output_tool(provider: &AIProvider, model: &str) -> bool {
model.contains("claude") || provider == &AIProvider::AWSBedrock
backend/windmill-api-auth/src/lib.rs
+9
View File
@@ -275,6 +275,15 @@ fn scope_restrictions(scopes: Option<&[String]>) -> Option<Vec<&String>> {
(!restrictions.is_empty()).then_some(restrictions)
}
/// True when the token carries no real scope restriction — unscoped, an empty scope
/// list, or only `if_jobs:filter_tags:` filters — so it holds the full privileges of
/// its user and can reach any non-job route they are authorized for (mirrors
/// `check_scopes` / `check_route_access`). A `false` result means the token is
/// genuinely scope-restricted.
pub fn is_effectively_unscoped(scopes: Option<&[String]>) -> bool {
scope_restrictions(scopes).is_none()
}
/// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes
/// a credential on behalf of `authed`: the resulting credential must never be
/// more privileged than the caller's own token.
backend/windmill-api-embeddings/src/lib.rs
+99 -28
View File
@@ -56,6 +56,10 @@ lazy_static::lazy_static! {
pub static ref EMBEDDINGS_DB: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
pub static ref MODEL_INSTANCE: Arc<RwLock<Option<Arc<ModelInstance>>>> = Arc::new(RwLock::new(None));
pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::<u64>().ok()).flatten().unwrap_or(3600 * 24);
// On a failed init/refresh we retry after this short interval instead of the
// full pulling interval, so a transient startup error doesn't leave the
// embeddings DB uninitialized for a whole day.
pub static ref HUB_EMBEDDINGS_RETRY_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_RETRY_INTERVAL_SECS").ok().map(|x| x.parse::<u64>().ok()).flatten().unwrap_or(60);
}
#[cfg(feature = "embedding")]
@@ -112,6 +116,26 @@ pub struct ResourceTypeResult {
score: f32,
schema: Option<serde_json::Value>,
}
/// Drop results whose score falls more than `max_relative_drop` below the best
/// match, so a strong hit isn't diluted by weakly-related entries that merely
/// clear the similarity floor. Expects `results` sorted by descending score and
/// a positive top score (guaranteed by the caller's similarity threshold).
#[cfg(feature = "embedding")]
fn trim_to_top_score<T>(
results: Vec<T>,
max_relative_drop: f32,
score: impl Fn(&T) -> f32,
) -> Vec<T> {
if results.len() <= 1 {
return results;
}
let top_score = score(&results[0]);
results
.into_iter()
.take_while(|r| (top_score - score(r)) / top_score <= max_relative_drop)
.collect()
}
#[cfg(feature = "embedding")]
async fn query_resource_types(
Query(query): Query<ResourceTypesQuery>,
@@ -511,15 +535,7 @@ impl EmbeddingsDb {
})
.collect();
let mut results = results?;
if results.len() > 1 {
let top_score = results[0].score;
results = results
.into_iter()
.take_while(|r| (top_score - r.score) / top_score <= 0.05)
.collect();
}
let results = trim_to_top_score(results?, 0.05, |r| r.score);
Ok(results)
}
@@ -560,7 +576,7 @@ impl EmbeddingsDb {
Some(0.75),
);
let results: Result<_> = results
let results: Result<Vec<ResourceTypeResult>> = results
.iter()
.map(|r| {
let metadata = r
@@ -582,7 +598,9 @@ impl EmbeddingsDb {
})
.collect();
results
let results = trim_to_top_score(results?, 0.05, |r| r.score);
Ok(results)
}
}
@@ -601,42 +619,67 @@ pub fn load_embeddings_db(db: &Pool<Postgres>) -> () {
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
let model_instance = ModelInstance::new().await;
if let Ok(model_instance) = model_instance {
let mut model_instance_lock = MODEL_INSTANCE.write().await;
*model_instance_lock = Some(Arc::new(model_instance));
drop(model_instance_lock);
loop {
update_embeddings_db(&db_clone).await;
tokio::time::sleep(std::time::Duration::from_secs(
*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS,
))
.await;
// Keep retrying model init: a transient failure here must not
// permanently disable embeddings until the next process restart.
// Backoff decays to the pulling interval so an environment where it
// can never succeed (e.g. air-gapped, embeddings left enabled)
// settles into ~1 attempt/interval rather than a tight error loop.
let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS;
loop {
match ModelInstance::new().await {
Ok(model_instance) => {
let mut model_instance_lock = MODEL_INSTANCE.write().await;
*model_instance_lock = Some(Arc::new(model_instance));
break;
}
Err(e) => {
tracing::error!(
"Failed to initialize model instance: {}. Retrying in {}s...",
e,
backoff_secs
);
tokio::time::sleep(std::time::Duration::from_secs(backoff_secs)).await;
backoff_secs = backoff_secs
.saturating_mul(2)
.min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS);
}
}
} else {
tracing::error!(
"Failed to initialize model instance: {}",
model_instance.err().unwrap()
);
}
let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS;
loop {
let sleep_secs = if update_embeddings_db(&db_clone).await {
backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS;
*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS
} else {
let secs = backoff_secs;
backoff_secs = backoff_secs
.saturating_mul(2)
.min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS);
secs
};
tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await;
}
});
}
}
#[cfg(feature = "embedding")]
pub async fn update_embeddings_db(db: &Pool<Postgres>) -> () {
pub async fn update_embeddings_db(db: &Pool<Postgres>) -> bool {
if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() {
tracing::info!("Creating embeddings DB...");
let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await;
if let Err(e) = new_embeddings_db.as_ref() {
tracing::error!("Failed to create embeddings db: {}", e);
false
} else {
let mut embeddings_db = EMBEDDINGS_DB.write().await;
*embeddings_db = new_embeddings_db.ok();
tracing::info!("Created embeddings DB");
true
}
} else {
tracing::error!("Could not update embeddings DB, model instance not initialized");
false
}
}
@@ -659,3 +702,31 @@ pub fn workspaced_service() -> Router {
pub fn global_service() -> Router {
Router::new()
}
#[cfg(all(test, feature = "embedding"))]
mod tests {
use super::trim_to_top_score;
#[test]
fn trims_scores_more_than_5pct_below_top() {
// top=1.0, cutoff at 0.95: 0.96 stays (0.04 drop), 0.93 is the first
// beyond the cutoff so take_while stops there and drops the tail.
let kept = trim_to_top_score(vec![1.0f32, 0.97, 0.96, 0.93, 0.9], 0.05, |s| *s);
assert_eq!(kept, vec![1.0, 0.97, 0.96]);
}
#[test]
fn keeps_all_when_tightly_clustered() {
let kept = trim_to_top_score(vec![0.9f32, 0.89, 0.88], 0.05, |s| *s);
assert_eq!(kept, vec![0.9, 0.89, 0.88]);
}
#[test]
fn passes_through_zero_or_one_result() {
assert_eq!(
trim_to_top_score(Vec::<f32>::new(), 0.05, |s| *s),
Vec::<f32>::new()
);
assert_eq!(trim_to_top_score(vec![0.42f32], 0.05, |s| *s), vec![0.42]);
}
}
backend/windmill-api-flows/src/flows.rs
+8 -5
View File
@@ -101,11 +101,7 @@ async fn list_search_flows(
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchFlow>> {
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let mut tx = user_db.begin(&authed).await?;
let allowed = build_scope_path_predicate(&authed, "flows", "read");
@@ -531,6 +527,11 @@ async fn create_flow(
}
check_scopes(&authed, || format!("flows:write:{}", nf.path))?;
// A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly.
// (The concurrency settings inside the flow value are normalized on deserialization; see
// ConcurrencySettings.) Runtime guards also protect already-stored rows.
nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout);
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
AuditAuthorable::username(&authed),
@@ -1016,7 +1017,9 @@ async fn update_flow(
}
let flow_path = flow_path.to_path();
// The URL identifies the flow being updated; the body path is only needed to rename.
let nf = ef.into_new_flow(flow_path);
let mut nf = ef.into_new_flow(flow_path);
// A `<= 0` flow timeout is "unset", not a 0-second limit (see create_flow).
nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout);
check_scopes(&authed, || format!("flows:write:{}", flow_path))?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
backend/windmill-api-groups/src/folders.rs
+15
View File
@@ -243,6 +243,7 @@ async fn create_folder(
Path(w_id): Path<String>,
Json(mut ng): Json<NewFolder>,
) -> Result<String> {
crate::check_demo_workspace_restriction(&authed, &w_id, "Folder creation")?;
if let Some(labels) = ng.labels.as_mut() {
dedup_labels(labels);
}
@@ -418,6 +419,12 @@ async fn update_folder(
return Err(Error::PermissionDenied(msg));
}
// update_folder can also grant permissions (owners / extra_perms / default_permissioned_as),
// so it is a sharing path and must honor the demo-workspace sharing restriction.
if ng.owners.is_some() || ng.extra_perms.is_some() || ng.default_permissioned_as.is_some() {
crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?;
}
let mut sqlb = SqlBuilder::update_table("folder");
sqlb.and_where_eq("name", "?".bind(&name));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
@@ -820,6 +827,7 @@ async fn add_owner(
Path((w_id, name)): Path<(String, String)>,
Json(Owner { owner, .. }): Json<Owner>,
) -> Result<String> {
crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?;
let mut tx = user_db.begin(&authed).await?;
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
@@ -884,6 +892,13 @@ async fn remove_owner(
Path((w_id, name)): Path<(String, String)>,
Json(Owner { owner, write }): Json<Owner>,
) -> Result<String> {
// remove_owner with a `write` value is a grant path: it jsonb_set's the owner's
// permission level into extra_perms (only write=None is a pure revoke), so the
// demo-workspace sharing restriction must apply when a level is being set.
if write.is_some() {
crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?;
}
let mut tx = user_db.begin(&authed).await?;
not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
backend/windmill-api-groups/src/granular_acls.rs
+1
View File
@@ -96,6 +96,7 @@ async fn add_granular_acl(
Path((w_id, path)): Path<(String, StripPath)>,
Json(GranularAcl { owner, write }): Json<GranularAcl>,
) -> Result<String> {
crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?;
let path = path.to_path();
let (kind, path) = path
backend/windmill-api-groups/src/groups.rs
+1
View File
@@ -234,6 +234,7 @@ async fn create_group(
Path(w_id): Path<String>,
Json(ng): Json<NewGroup>,
) -> Result<String> {
crate::check_demo_workspace_restriction(&authed, &w_id, "Group creation")?;
let mut tx = user_db.begin(&authed).await?;
check_name_conflict(&mut tx, &w_id, &ng.name).await?;
backend/windmill-api-groups/src/lib.rs
+20
View File
@@ -2,3 +2,23 @@ pub mod folder_history;
pub mod folders;
pub mod granular_acls;
pub mod groups;
use windmill_api_auth::ApiAuthed;
use windmill_common::{error::Error, worker::CLOUD_HOSTED};
/// The public demo workspace on the managed cloud is kept clean and consistent by
/// restricting folder creation, item sharing, and group creation for non-admins.
/// `action` is a short noun phrase completing "… is disabled …" (e.g.
/// "Folder creation", "Sharing"). Returns `Err(BadRequest)` when the caller is blocked.
pub fn check_demo_workspace_restriction(
authed: &ApiAuthed,
w_id: &str,
action: &str,
) -> Result<(), Error> {
if *CLOUD_HOSTED && w_id == "demo" && !authed.is_admin {
return Err(Error::BadRequest(format!(
"{action} is disabled in the demo workspace. Create your own workspace to keep the demo clean and consistent."
)));
}
Ok(())
}
backend/windmill-api-integration-tests/tests/resources.rs
+13
View File
@@ -275,6 +275,19 @@ async fn test_resource_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
let body = resp.json::<serde_json::Value>().await?;
assert_eq!(body["description"], "Updated description");
// --- update (resource_type) ---
// An update that only changes resource_type must persist it.
let resp = authed(client().post(resource_url(port, "update", "u/test-user/new_resource")))
.json(&json!({"resource_type": "mcp_server"}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
let resp = authed_get(port, "get", "u/test-user/new_resource").await;
let body = resp.json::<serde_json::Value>().await?;
assert_eq!(body["resource_type"], "mcp_server");
// --- update_value ---
let resp = authed(client().post(resource_url(
port,
backend/windmill-api-integration-tests/tests/workspaces.rs
+58
View File
@@ -901,6 +901,64 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row(
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_error_handler_instance_alerts_fallback(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces");
let stored = || async {
sqlx::query_scalar!(
"SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'"
)
.fetch_one(&db)
.await
};
let resp = authed(client().post(format!("{base}/edit_error_handler")))
.json(&json!({"path": null, "fallback_to_instance_alerts": true}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "enable: {}", resp.text().await?);
assert!(stored().await?);
// A client that predates the setting (the CLI pushing settings.yaml) omits the field and
// must not silently turn it back off.
let resp = authed(client().post(format!("{base}/edit_error_handler")))
.json(&json!({"path": null, "extra_args": null}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "omitted: {}", resp.text().await?);
assert!(stored().await?);
sqlx::query!(
"UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'"
)
.execute(&db)
.await?;
let resp = authed(client().post(format!("{base}/edit_error_handler")))
.json(&json!({"path": null, "fallback_to_instance_alerts": true}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 400, "fork must be rejected");
// The settings page stops offering the option once the workspace is a fork, so its next save
// sends `false`: that must go through rather than lock the whole error handler behind a 400.
let resp = authed(client().post(format!("{base}/edit_error_handler")))
.json(&json!({"path": null, "fallback_to_instance_alerts": false}))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?);
assert!(!stored().await?);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_get_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
backend/windmill-api-scripts/src/scripts.rs
+7 -4
View File
@@ -157,12 +157,8 @@ async fn list_search_scripts(
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchScript>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 10000;
#[cfg(not(feature = "enterprise"))]
let n = 10;
let allowed = build_scope_path_predicate(&authed, "scripts", "read");
let rows = sqlx::query_as!(
SearchScript,
@@ -918,6 +914,13 @@ async fn create_script_internal<'c>(
}
check_scopes(&authed, || format!("scripts:write:{}", ns.path))?;
// Normalize positive-only settings so a `<= 0` value (e.g. a CLI-pushed `0`) persists as
// disabled rather than as a zero-slot concurrency cap or a 0-second timeout. Deserialization
// already normalizes the concurrency fields; re-applying here also covers `timeout` and any
// NewScript built in-process rather than from a request body.
ns.timeout = windmill_common::runnable_settings::none_if_non_positive(ns.timeout);
ns.concurrency_settings = ns.concurrency_settings.normalized();
guard_script_from_debounce_data(&ns).await?;
let codebase = ns.codebase.as_ref();
backend/windmill-api-settings/src/lib.rs
+75 -19
View File
@@ -32,11 +32,9 @@ use windmill_common::DB;
use ee_oss::validate_license_key;
use windmill_common::usernames::generate_instance_username_for_all_users;
#[cfg(feature = "enterprise")]
use axum::extract::Query;
use axum::{
body::Body,
extract::{Extension, Path},
extract::{Extension, Path, Query},
response::Response,
routing::{get, post},
Json, Router,
@@ -283,15 +281,15 @@ pub async fn test_s3_bucket(
let mut list = client.list(Some(
&windmill_object_store::object_store_reexports::Path::from("".to_string()),
));
let first_file = list.next().await;
if first_file.is_some() {
if let Err(e) = first_file.as_ref().unwrap() {
match list.next().await {
Some(Err(e)) => {
tracing::error!("error listing bucket: {e:#}");
error::Error::internal_err(format!("Failed to list files in blob storage: {e:#}"));
return Err(error::Error::internal_err(format!(
"Failed to list files in blob storage: {e:#}"
)));
}
tracing::info!("Listed files: {:?}", first_file.unwrap());
} else {
tracing::info!("No files in blob storage");
Some(Ok(first_file)) => tracing::info!("Listed files: {:?}", first_file),
None => tracing::info!("No files in blob storage"),
}
let path = windmill_object_store::object_store_reexports::Path::from(format!(
@@ -1580,6 +1578,7 @@ async fn refresh_custom_instance_user_pwd(
) -> JsonResult<()> {
require_super_admin(&db, &authed.email).await?;
windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?;
windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?;
Ok(Json(()))
}
@@ -1710,11 +1709,23 @@ async fn setup_custom_instance_pg_database_inner(
))
})?;
// The replication attribute lives on a dedicated role used by postgres trigger
// connections. The getter creates the role (with its stored password) when the
// migration couldn't.
if let Err(e) = windmill_common::utils::get_custom_pg_instance_replication_password(db).await {
tracing::error!("Failed to ensure custom_instance_replication_user exists: {e:#}");
}
if let Err(e) = client
.batch_execute(&format!("ALTER ROLE custom_instance_user REPLICATION;"))
.batch_execute(
"ALTER ROLE custom_instance_replication_user REPLICATION;
GRANT custom_instance_user TO custom_instance_replication_user;
ALTER ROLE custom_instance_user NOREPLICATION;",
)
.await
{
tracing::error!("Failed to grant replication permission to custom_instance_user: {e:#}");
tracing::error!(
"Failed to grant replication permission to custom_instance_replication_user: {e:#}"
);
}
logs.grant_permissions = "OK".to_string();
@@ -1976,25 +1987,53 @@ async fn fetch_resource_types_from_hub() -> error::Result<Vec<CachedResourceType
.collect())
}
#[derive(serde::Deserialize)]
struct SyncResourceTypesQuery {
name: Option<String>,
}
async fn sync_cached_resource_types(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Query(SyncResourceTypesQuery { name }): Query<SyncResourceTypesQuery>,
) -> error::Result<String> {
require_super_admin(&db, &authed.email).await?;
use windmill_common::worker::HUB_RT_CACHE_DIR;
let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR);
let cached_types = match tokio::fs::read_to_string(&cache_path).await {
Ok(content) => serde_json::from_str::<Vec<CachedResourceType>>(&content).map_err(|e| {
error::Error::InternalErr(format!("Failed to parse cached resource types: {}", e))
})?,
Err(_) => fetch_resource_types_from_hub().await?,
// Manual sync is hub-first so it lands newly-published hub types on demand. The
// on-disk cache is only a fallback for when the hub is unreachable (airgapped
// installs / network error); refreshing it is left to the daily cache-rt cron and
// the startup sync in main.rs, which own the offline path.
let (resource_types, from_hub) = match fetch_resource_types_from_hub().await {
Ok(types) => {
tracing::info!("Fetched {} resource types live from the hub", types.len());
(types, true)
}
Err(hub_err) => {
tracing::warn!(
"Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}"
);
match tokio::fs::read_to_string(&cache_path).await {
Ok(content) => {
let parsed = serde_json::from_str::<Vec<CachedResourceType>>(&content)
.map_err(|e| {
error::Error::InternalErr(format!(
"Failed to parse cached resource types: {}",
e
))
})?;
(parsed, false)
}
Err(_) => return Err(hub_err),
}
}
};
let mut synced_count = 0;
for rt in &cached_types {
for rt in &resource_types {
let exists: Option<bool> = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)",
&rt.name,
@@ -2023,10 +2062,27 @@ async fn sync_cached_resource_types(
synced_count += 1;
}
// If a specific type was requested and is still absent after syncing, surface an
// explicit not-found instead of a silent "Synced 0". Word it by source so the
// cache-fallback path does not claim it checked the hub.
if let Some(name) = name.as_deref() {
if !resource_types.iter().any(|rt| rt.name == name) {
let source = if from_hub {
"on the hub"
} else {
"in the cached resource types (hub unreachable)"
};
return Err(error::Error::NotFound(format!(
"resource type '{}' not found {}",
name, source
)));
}
}
Ok(format!(
"Synced {} resource types ({} unchanged)",
synced_count,
cached_types.len() - synced_count
resource_types.len() - synced_count
))
}
backend/windmill-api-workspaces/src/workspaces.rs
+224 -30
View File
@@ -303,6 +303,7 @@ pub struct WorkspaceSettings {
pub success_handler: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub public_app_execution_limit_per_minute: Option<i32>,
pub error_handler_fallback_to_instance_alerts: bool,
}
/// Subset of `WorkspaceSettings` that is safe to return to any workspace
@@ -452,6 +453,8 @@ struct CreateWorkspace {
name: String,
username: Option<String>,
color: Option<String>,
#[serde(default)]
error_handler_fallback_to_instance_alerts: bool,
}
#[derive(Deserialize)]
@@ -559,6 +562,9 @@ pub struct EditErrorHandlerNew {
pub muted_on_cancel: bool,
#[serde(default)]
pub muted_on_user_path: bool,
/// Left as `None` by clients that predate the setting (the CLI among them), which must
/// keep the stored value rather than silently reset it on every settings push.
pub fallback_to_instance_alerts: Option<bool>,
}
// Legacy format for error handler (flat fields from old CLI)
@@ -587,6 +593,7 @@ impl EditErrorHandler {
extra_args: legacy.error_handler_extra_args,
muted_on_cancel: legacy.error_handler_muted_on_cancel,
muted_on_user_path: false, // Old format doesn't have this field
fallback_to_instance_alerts: None,
},
}
}
@@ -827,21 +834,37 @@ async fn reject_dev_label_matching_tracked_branch(
Ok(())
}
/// Reject parent-only git-sync settings on a fork workspace. Auto-pull, fork
/// PRs, and promotion mode are all configured at the parent: repo → fork sync is
/// routed by the parent's webhook/poller (`sync_forks`), a fork-owned auto-pull
/// would register a second webhook on the same GitHub repo per fork, and a
/// fork's deploys always go to its `wm-fork/**` branch so a promotion repo could
/// never take effect there.
/// Reject parent-only git-sync settings on a fork workspace. Auto-pull and fork
/// PRs are configured at the parent: repo → fork sync is routed by the parent's
/// webhook/poller (`sync_forks`), and a fork-owned auto-pull would register a
/// second webhook on the same GitHub repo per fork. Promotion mode is rejected
/// on throwaway forks (their deploys always go to their `wm-fork/**` branch, so
/// a promotion repo could never take effect) but allowed on a **dev workspace**,
/// which deploys per-item `wm_deploy/**` branches that promote into the parent.
async fn reject_parent_only_git_sync_settings_on_fork<'a>(
db: &DB,
w_id: &str,
mut repos: impl Iterator<Item = &'a windmill_common::workspaces::GitRepositorySettings>,
repos: impl Iterator<Item = &'a windmill_common::workspaces::GitRepositorySettings>,
) -> Result<()> {
let offending = repos.find_map(|r| {
let row = sqlx::query!(
"SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1",
w_id
)
.fetch_optional(db)
.await?;
let is_fork = row
.as_ref()
.and_then(|r| r.parent_workspace_id.as_ref())
.is_some()
|| w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX);
if !is_fork {
return Ok(());
}
let is_dev = row.map(|r| r.is_dev_workspace).unwrap_or(false);
let offending = repos.into_iter().find_map(|r| {
if r.auto_pull.as_ref().is_some_and(|a| a.enabled) {
Some("Auto-pull")
} else if r.use_individual_branch.unwrap_or(false) {
} else if r.use_individual_branch.unwrap_or(false) && !is_dev {
Some("Promotion mode")
} else if r.fork_open_prs {
Some("Opening PRs for fork deploys")
@@ -849,17 +872,7 @@ async fn reject_parent_only_git_sync_settings_on_fork<'a>(
None
}
});
let Some(offending) = offending else {
return Ok(());
};
let parent = sqlx::query_scalar!(
"SELECT parent_workspace_id FROM workspace WHERE id = $1",
w_id
)
.fetch_optional(db)
.await?
.flatten();
if parent.is_some() || w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX) {
if let Some(offending) = offending {
return Err(Error::BadRequest(format!(
"{offending} cannot be configured on a fork workspace: it is managed from the parent workspace's git sync settings"
)));
@@ -968,7 +981,8 @@ async fn get_settings(
auto_invite,
error_handler,
success_handler,
public_app_execution_limit_per_minute
public_app_execution_limit_per_minute,
error_handler_fallback_to_instance_alerts
FROM
workspace_settings
WHERE
@@ -3314,6 +3328,95 @@ async fn check_open_prs_license<'a>(
Ok(())
}
/// Promotion mode (`use_individual_branch`: per-item `wm_deploy/**` deploy
/// branches) is an EE feature; runtime-gate it like auto-pull and PR creation
/// so an enterprise binary without an active plan can't enable it via either
/// git-sync edit endpoint.
#[cfg(feature = "enterprise")]
async fn check_promotion_license<'a>(
mut repos: impl Iterator<Item = &'a windmill_common::workspaces::GitRepositorySettings>,
) -> Result<()> {
if repos.any(|r| r.use_individual_branch.unwrap_or(false)) {
check_git_sync_ee_license("Promotion mode").await?;
}
Ok(())
}
/// Promotion on a dev workspace needs the dev-aware sync script (hub >= 28796):
/// an older pinned script bundles a CLI that force-disables per-item branches
/// on every fork, so enabling promotion would silently keep deploying to the
/// env-label branch. Reject with an actionable error instead (the dispatcher
/// demotes inherited configs the same way). Roots run promotion on any script
/// version, and auto-managed repositories (no pin) always use the latest.
#[cfg(feature = "enterprise")]
async fn check_dev_promotion_script_version<'a>(
db: &DB,
w_id: &str,
repos: impl Iterator<Item = &'a windmill_common::workspaces::GitRepositorySettings>,
) -> Result<()> {
let mut offending: Option<String> = None;
for r in repos {
if !r.use_individual_branch.unwrap_or(false) {
continue;
}
if !r.is_script_meets_min_version(28796)? {
offending = Some(r.effective_script_path().to_string());
break;
}
}
let Some(offending) = offending else {
return Ok(());
};
let is_dev = sqlx::query!(
"SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1",
w_id
)
.fetch_optional(db)
.await?
.map(|r| r.is_dev_workspace)
.unwrap_or(false);
if !is_dev {
return Ok(());
}
Err(Error::BadRequest(format!(
"Promotion mode on a dev workspace requires git sync script version 28796 or newer, \
but this repository pins '{offending}'. Update the pinned sync script (or reset it to \
auto-managed) first."
)))
}
/// A dev workspace's promotion must target its parent ("prod") workspace's own
/// git repository (same URL and branch) — that is what "promote to prod" means.
/// A fork-created dev inherits prod's repo; an **attached** dev keeps its own,
/// which may be unrelated. Reject enabling promotion on a repo the parent does
/// not track so the UI can't present an unrelated repo as prod's target. The
/// deploy path re-checks the same invariant (a resource edit could break it
/// after save), via the shared `dev_promotion_target_matches_parent`.
#[cfg(all(feature = "enterprise", feature = "private"))]
async fn check_dev_promotion_targets_parent_repo<'a>(
db: &DB,
w_id: &str,
repos: impl Iterator<Item = &'a windmill_common::workspaces::GitRepositorySettings>,
) -> Result<()> {
for r in repos.filter(|r| r.use_individual_branch.unwrap_or(false)) {
if !windmill_common::git_sync_ee::dev_promotion_target_matches_parent(
db,
w_id,
&r.git_repo_resource_path,
)
.await?
{
return Err(Error::BadRequest(
"Promotion mode on a dev workspace must reuse the parent workspace's git repository \
(same URL and branch), but this repository is not one the parent tracks — promotion \
would target a repository the parent does not sync with."
.to_string(),
));
}
}
Ok(())
}
#[cfg(feature = "enterprise")]
async fn check_git_sync_access(_db: &DB, _w_id: &str) -> Result<()> {
Ok(())
@@ -3458,6 +3561,25 @@ async fn edit_git_sync_config(
}
#[cfg(feature = "enterprise")]
check_open_prs_license(git_sync_settings.repositories.iter()).await?;
#[cfg(feature = "enterprise")]
check_promotion_license(git_sync_settings.repositories.iter()).await?;
#[cfg(feature = "enterprise")]
check_dev_promotion_script_version(&db, &w_id, git_sync_settings.repositories.iter())
.await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
check_dev_promotion_targets_parent_repo(&db, &w_id, git_sync_settings.repositories.iter())
.await?;
// Promotion mode: EE only (mirrors edit_git_sync_repository).
#[cfg(not(feature = "enterprise"))]
if git_sync_settings
.repositories
.iter()
.any(|r| r.use_individual_branch.unwrap_or(false))
{
return Err(Error::BadRequest(
"Promotion mode is an Enterprise Edition feature".to_string(),
));
}
// Preserve server-owned auto-pull state (webhook id/secret, synced sha, last
// status) that the redacted GET response omits — otherwise a whole-config
// save from the UI would drop the webhook secret (breaking delivery) or
@@ -3593,7 +3715,8 @@ async fn edit_git_sync_config(
tracing::warn!("git auto-pull: webhook field persist error: {}", e);
}
for (path, hook_id) in removed_webhooks {
if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url(&db, &w_id, &path).await
if let Ok(url) =
windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &w_id, &path).await
{
let _ =
windmill_common::git_sync_ee::delete_repo_webhook(&db, &w_id, &url, hook_id)
@@ -3670,6 +3793,13 @@ async fn edit_git_sync_repository(
}
#[cfg(feature = "enterprise")]
check_open_prs_license(std::iter::once(&new_config.repository)).await?;
#[cfg(feature = "enterprise")]
check_promotion_license(std::iter::once(&new_config.repository)).await?;
#[cfg(feature = "enterprise")]
check_dev_promotion_script_version(&db, &w_id, std::iter::once(&new_config.repository)).await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
check_dev_promotion_targets_parent_repo(&db, &w_id, std::iter::once(&new_config.repository))
.await?;
// Promotion mode: EE only
#[cfg(not(feature = "enterprise"))]
@@ -3945,7 +4075,7 @@ async fn delete_git_sync_repository(
// Removal is durable now — best-effort delete the GitHub webhook.
#[cfg(all(feature = "enterprise", feature = "private"))]
if let Some(hook_id) = webhook_to_delete {
if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url(
if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url_interpolated(
&db,
&w_id,
&request.git_repo_resource_path,
@@ -4244,6 +4374,19 @@ async fn edit_error_handler(
let mut tx = db.begin().await?;
if let Some(fallback_to_instance_alerts) = ee.fallback_to_instance_alerts {
if fallback_to_instance_alerts {
ensure_instance_alert_fallback_allowed(&mut tx, &w_id).await?;
}
sqlx::query!(
"UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2",
fallback_to_instance_alerts,
&w_id
)
.execute(&mut *tx)
.await?;
}
sqlx::query_as!(
Group,
"INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING",
@@ -4322,7 +4465,16 @@ async fn edit_error_handler(
ActionKind::Update,
&w_id,
Some(&authed.email),
Some([("error_handler", &format!("{:?}", ee.path)[..])].into()),
Some(
[
("error_handler", &format!("{:?}", ee.path)[..]),
(
"fallback_to_instance_alerts",
&format!("{:?}", ee.fallback_to_instance_alerts)[..],
),
]
.into(),
),
)
.await?;
tx.commit().await?;
@@ -4799,6 +4951,36 @@ async fn session_workspace_status(
Ok(Json(statuses))
}
/// The instance critical alert channels belong to the instance operator, who on cloud is
/// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run
/// throwaway copies of their parent's runnables, so instance-wide operational alerting must
/// stay a property of the real workspace.
async fn ensure_instance_alert_fallback_allowed<'c>(
tx: &mut Transaction<'c, Postgres>,
w_id: &str,
) -> Result<()> {
if *CLOUD_HOSTED {
return Err(Error::BadRequest(
"Reporting to the instance critical alert channels is not available on cloud"
.to_string(),
));
}
let is_fork = sqlx::query_scalar!(
r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#,
w_id
)
.fetch_optional(&mut **tx)
.await?
.unwrap_or(false);
if is_fork {
return Err(Error::BadRequest(
"Reporting to the instance critical alert channels cannot be enabled on a fork workspace"
.to_string(),
));
}
Ok(())
}
pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> {
if w_id == "global" {
return Err(windmill_common::error::Error::BadRequest(
@@ -4975,12 +5157,16 @@ async fn create_workspace(
)
.execute(&mut *tx)
.await?;
if nw.error_handler_fallback_to_instance_alerts {
ensure_instance_alert_fallback_allowed(&mut tx, &nw.id).await?;
}
sqlx::query!(
"INSERT INTO workspace_settings
(workspace_id, color)
VALUES ($1, $2)",
(workspace_id, color, error_handler_fallback_to_instance_alerts)
VALUES ($1, $2, $3)",
nw.id,
nw.color,
nw.error_handler_fallback_to_instance_alerts,
)
.execute(&mut *tx)
.await?;
@@ -6557,7 +6743,7 @@ async fn enforce_fork_depth(
/// True if `raw` (the text form of a `json` value) contains a genuine `\u0000`
/// NUL escape: a `u0000` preceded by an ODD run of backslashes. Mirrors the
/// parity rule in `strip_null_chars` (windmill-api `apps.rs`) — an even run
/// parity rule in `windmill_common::utils::strip_json_nul` — an even run
/// (`\\u0000`) is an escaped backslash then the literal text "u0000" (common in
/// minified JS) and is jsonb-safe. A genuine NUL is exactly what the
/// `json`→`jsonb` re-encode in `clone_apps` / `clone_flows` rejects with
@@ -7097,8 +7283,12 @@ async fn attach_dev_workspace(
)
.execute(&mut *tx)
.await?;
// Clearing the instance-alert opt-in here keeps the stored setting truthful for a workspace
// that becomes parent-managed: dispatch enforces the fork boundary on its own, but a lingering
// `true` would survive a later detach and would make the settings page submit a value the API
// rejects on a fork.
sqlx::query!(
"UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2",
"UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2",
&prod_w_id,
&dev_w_id
)
@@ -7173,7 +7363,8 @@ async fn attach_dev_workspace(
// (their auto_pull is gone), so remove them from GitHub.
#[cfg(all(feature = "enterprise", feature = "private"))]
for (path, hook_id) in stripped_webhooks {
if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url(&db, &dev_w_id, &path).await
if let Ok(url) =
windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &dev_w_id, &path).await
{
let _ =
windmill_common::git_sync_ee::delete_repo_webhook(&db, &dev_w_id, &url, hook_id)
@@ -7210,7 +7401,8 @@ async fn attach_dev_workspace(
}
/// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the
/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork).
/// prod lock. Whether `parent_workspace_id` is kept depends on the workspace's origin (see the
/// UPDATE below): a genuine fork stays a fork, a standalone workspace returns to standalone.
async fn detach_dev_workspace(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -10109,6 +10301,8 @@ const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[
("ai_session", "deployed"),
("ai_session", "archived"),
("ai_session", "deleted"),
("ai_session", "beta_optout"),
("ai_session", "beta_optin"),
("ai_chat", "message"),
("ai_chat", "model"),
("ai_chat", "tool"),
backend/windmill-api-workspaces/src/workspaces_extra.rs
+3 -2
View File
@@ -108,7 +108,7 @@ pub(crate) async fn change_workspace_id(
// Duplicate workspace settings (keep copy in old workspace for reference)
info!("Duplicating workspace_settings table");
sqlx::query!(
"INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler FROM workspace_settings WHERE workspace_id = $2",
"INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2",
&rw.new_id,
&old_id
)
@@ -805,7 +805,8 @@ pub(crate) async fn change_workspace_id(
#[cfg(all(feature = "enterprise", feature = "private"))]
for (path, hook_id) in stale_webhooks {
if let Ok(url) =
windmill_common::git_sync_ee::resolve_repo_url(&db, &rw.new_id, &path).await
windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &rw.new_id, &path)
.await
{
let _ =
windmill_common::git_sync_ee::delete_repo_webhook(&db, &rw.new_id, &url, hook_id)
backend/windmill-api/Cargo.toml
+5 -1
View File
@@ -26,7 +26,11 @@ kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"]
kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"]
nats = ["dep:windmill-trigger-nats", "windmill-store/nats"]
websocket = ["dep:windmill-trigger-websocket"]
smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email"]
smtp = ["instance_smtp", "dep:mail-parser", "dep:openssl", "dep:windmill-trigger-email"]
# Outbound instance-SMTP email (the send_email_with_instance_smtp endpoint and
# critical alerts) without the inbound email trigger's openssl/mail-parser deps.
# `smtp` is the full trigger + endpoint; `instance_smtp` is the endpoint only.
instance_smtp = ["windmill-common/smtp"]
license = ["dep:rsa", "windmill-api-settings/license"]
zip = ["dep:async_zip"]
oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"]
backend/windmill-api/openapi.yaml
+768 -2
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.764.0
version: 1.769.0
title: Windmill API
contact:
@@ -1330,7 +1330,7 @@ paths:
/settings/refresh_custom_instance_user_pwd:
post:
summary: Refreshes the password for the custom_instance_user
summary: Refreshes the passwords for the custom_instance_user and the custom_instance_replication_user (used by postgres triggers)
operationId: refreshCustomInstanceUserPwd
tags:
- setting
@@ -3617,6 +3617,9 @@ paths:
public_app_execution_limit_per_minute:
type: integer
description: Rate limit for public app executions per minute per server. NULL or 0 means disabled.
error_handler_fallback_to_instance_alerts:
type: boolean
description: Report failed jobs to the instance critical alert channels when no workspace error handler is set.
/w/{workspace}/workspaces/get_deploy_to:
get:
@@ -22963,6 +22966,472 @@ paths:
schema:
type: string
/w/{workspace}/hub/publish_draft:
post:
summary: create or update a hub project draft
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubDraft
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishDraftBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/scripts:
post:
summary: publish a script to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubScript
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishScriptBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/flows:
post:
summary: publish a flow to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubFlow
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishFlowBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/apps:
post:
summary: publish an app to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubApp
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishAppBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/raw_apps:
post:
summary: publish a raw app to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubRawApp
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishRawAppBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/raw_apps/{id}/embed:
post:
summary: set or clear the embed url of a hub raw app
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubRawAppEmbed
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: id
in: path
required: true
description: hub id of the raw app
schema:
type: integer
format: int64
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/RawAppEmbedBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/scripts/{ask_id}/recording:
post:
summary: attach a recording to a hub script
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubScriptRecording
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: ask_id
in: path
required: true
description: hub ask id of the script
schema:
type: integer
format: int64
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/RecordingBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/flows/{flow_id}/recording:
post:
summary: attach a recording to a hub flow
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubFlowRecording
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: flow_id
in: path
required: true
description: hub id of the flow
schema:
type: integer
format: int64
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/RecordingBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/projects/{slug}/pipeline_recording:
post:
summary: attach a data-pipeline recording to a hub project
description: |
Requires the caller to be a workspace admin. A data-pipeline recording is
scoped to the whole project (a folder cascade), not a single item. Forwards
the request to the configured Hub scoped to the `{workspace}:{folder}`
source and returns the Hub's status code and raw response body.
operationId: publishHubPipelineRecording
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: slug
in: path
required: true
description: hub project slug
schema:
$ref: "#/components/schemas/HubProjectSlug"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PipelineRecordingBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/resource_types:
post:
summary: publish a resource type to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubResourceType
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishResourceTypeBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/resources:
post:
summary: publish resource placeholders to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubResources
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishResourcesBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/triggers:
post:
summary: publish triggers to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubTriggers
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishTriggersBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/migrations:
post:
summary: publish data table migrations to a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: publishHubMigrations
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/PublishMigrationsBody"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/projects/{slug}/export:
get:
summary: export a hub project
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub and returns the Hub's status code and raw response body.
The folder scope is only needed to re-export the caller's own draft;
approved projects are public, so it is optional here.
operationId: getHubProjectExport
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: slug
in: path
required: true
description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen)
schema:
type: string
minLength: 3
maxLength: 50
pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$"
- name: folder
in: query
required: false
description: folder scoping the Hub project source (`{workspace}:{folder}`)
schema:
type: string
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/projects/{slug}/submit:
post:
summary: submit a hub project draft for review
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: submitHubProject
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: slug
in: path
required: true
description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen)
schema:
type: string
minLength: 3
maxLength: 50
pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$"
- $ref: "#/components/parameters/HubPublishFolder"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
/w/{workspace}/hub/project:
get:
summary: get the hub project linked to a workspace folder
description: |
Requires the caller to be a workspace admin. Forwards the request to the
configured Hub scoped to the `{workspace}:{folder}` source and returns
the Hub's status code and raw response body.
operationId: getHubProjectBySource
tags:
- hubPublish
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/HubPublishFolder"
responses:
"200":
description: raw Hub response body (status code is passed through from the Hub)
content:
text/plain:
schema:
type: string
components:
securitySchemes:
bearerAuth:
@@ -23011,6 +23480,16 @@ components:
required: true
schema:
type: string
HubPublishFolder:
name: folder
in: query
required: true
description: |
workspace folder scoping the Hub publication: a workspace can publish
one Hub project per folder and the Hub-side source key is
`{workspace}:{folder}`
schema:
type: string
PublicationName:
name: publication
in: path
@@ -23824,6 +24303,9 @@ components:
muted_on_user_path:
type: boolean
default: false
fallback_to_instance_alerts:
type: boolean
description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Omit to leave the stored value untouched.
EditErrorHandlerLegacy:
type: object
@@ -26475,6 +26957,7 @@ components:
- slack
- teams
- email
- instance_alerts
NewSchedule:
type: object
@@ -29403,6 +29886,10 @@ components:
type: string
color:
type: string
error_handler_fallback_to_instance_alerts:
type: boolean
default: false
description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Not available on cloud or on fork workspaces.
required:
- id
- name
@@ -31943,3 +32430,282 @@ components:
- name
- owner
- private
HubProjectSlug:
type: string
description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen)
minLength: 3
maxLength: 50
pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$"
PublishDraftBody:
type: object
properties:
slug:
$ref: "#/components/schemas/HubProjectSlug"
name:
type: string
summary:
type: string
readme:
type: string
required:
- slug
- name
- summary
PublishScriptBody:
type: object
properties:
summary:
type: string
app:
type: string
description:
type: string
kind:
type: string
content:
type: string
language:
type: string
schema:
type: object
lockfile:
type: string
path:
type: string
source_path:
type: string
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- summary
- app
- content
- language
- project_slug
PublishFlowInner:
type: object
properties:
summary:
type: string
description:
type: string
value:
type: object
schema:
type: object
required:
- summary
- value
PublishFlowBody:
type: object
properties:
flow:
$ref: "#/components/schemas/PublishFlowInner"
apps:
type: array
items:
type: string
path:
type: string
source_path:
type: string
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- flow
- apps
- project_slug
PublishAppBody:
type: object
properties:
app:
type: object
apps:
type: array
items:
type: string
description:
type: string
summary:
type: string
path:
type: string
source_path:
type: string
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- app
- apps
- summary
- project_slug
PublishRawAppBody:
type: object
properties:
raw:
type: string
apps:
type: array
items:
type: string
description:
type: string
summary:
type: string
path:
type: string
source_path:
type: string
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- raw
- apps
- summary
- project_slug
RawAppEmbedBody:
type: object
properties:
external_embed_url:
type: string
nullable: true
description: explicit `null` clears the embed (unpublish)
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- project_slug
RecordingBody:
type: object
properties:
recording:
type: object
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- project_slug
PipelineRecordingBody:
type: object
properties:
recording:
type: object
PublishResourceTypeBody:
type: object
properties:
name:
type: string
schema:
type: object
description:
type: string
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- name
- project_slug
PublishResourceBody:
type: object
properties:
path:
type: string
resource_type:
type: string
required:
- path
- resource_type
PublishResourcesBody:
type: object
properties:
resources:
type: array
items:
$ref: "#/components/schemas/PublishResourceBody"
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- resources
- project_slug
PublishTriggerBody:
type: object
properties:
path:
type: string
kind:
type: string
summary:
type: string
nullable: true
description:
type: string
nullable: true
config:
type: object
script_ask_id:
type: integer
format: int64
nullable: true
flow_id:
type: integer
format: int64
nullable: true
required:
- path
- kind
- config
PublishTriggersBody:
type: object
properties:
triggers:
type: array
items:
$ref: "#/components/schemas/PublishTriggerBody"
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- triggers
- project_slug
PublishMigrationBody:
type: object
description: one best-effort data table migration attached to a project (per data table)
properties:
datatable_name:
type: string
sql:
type: string
sql_down:
type: string
description: defaults to an empty string when omitted
enabled:
type: boolean
required:
- datatable_name
- sql
- enabled
PublishMigrationsBody:
type: object
properties:
migrations:
type: array
items:
$ref: "#/components/schemas/PublishMigrationBody"
project_slug:
$ref: "#/components/schemas/HubProjectSlug"
required:
- migrations
- project_slug
backend/windmill-api/src/ai.rs
+80 -34
View File
@@ -104,18 +104,7 @@ lazy_static::lazy_static! {
}
};
pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
.timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS))
.pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST)
.pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS)))
// The SSRF check in `get_base_url` only validates the configured `base_url`.
// reqwest follows up to 10 redirects by default and does not revalidate the
// hops, so a public base_url could 3xx the server into a private/internal
// address. Disable redirect following so the validated host is the only one
// we ever connect to. AI APIs respond directly and do not rely on redirects,
// so this holds even for ALLOW_PRIVATE_AI_BASE_URLS deployments.
.redirect(reqwest::redirect::Policy::none())
.user_agent("windmill/beta"))
pub(crate) static ref HTTP_CLIENT: Client = ai_http_client_builder()
.build()
.expect("Failed to build AI HTTP client - check system TLS configuration");
@@ -129,6 +118,65 @@ pub(crate) fn invalidate_ai_request_cache_for_workspace(workspace_id: &str) {
AI_REQUEST_CACHE.retain(|(cached_workspace_id, _), _| cached_workspace_id != workspace_id);
}
/// Shared configuration for every outbound AI HTTP client (the pooled
/// [`HTTP_CLIENT`] and the per-request DNS-pinned clients).
///
/// Redirects are disabled: the SSRF check only validates the configured host, so
/// a public host that 3xx-es could otherwise bounce us to a private/internal
/// address. AI APIs respond directly and do not rely on redirects, so this holds
/// even for ALLOW_PRIVATE_AI_BASE_URLS deployments. DNS pinning likewise only
/// covers the original host, so following a redirect would reopen the hole.
fn ai_http_client_builder() -> reqwest::ClientBuilder {
configure_client(
reqwest::ClientBuilder::new()
.timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS))
.pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST)
.pool_idle_timeout(Some(std::time::Duration::from_secs(
HTTP_POOL_IDLE_TIMEOUT_SECS,
)))
.redirect(reqwest::redirect::Policy::none())
.user_agent("windmill/beta"),
)
}
/// Build the client for a single outbound AI request to `url`, pinning DNS to
/// the SSRF-validated address so the connect cannot rebind to an internal IP
/// after the check (DNS-rebinding TOCTOU).
///
/// Returns the shared pooled [`HTTP_CLIENT`] unchanged when there is nothing to
/// pin — an IP-literal host, or a deployment that opted into private AI
/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint
/// as `get_base_url` apply, so the guard here is consistent with save-time
/// validation while additionally closing the connect-time window.
async fn pinned_ai_client_for(url: &str) -> Result<std::borrow::Cow<'static, Client>> {
use std::borrow::Cow;
use windmill_common::ssrf::SsrfValidationError;
if *windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS {
return Ok(Cow::Borrowed(&HTTP_CLIENT));
}
let target = windmill_common::ssrf::validate_url_for_ssrf(url)
.await
.map_err(|e| match e {
e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!(
"{e}. If you need to use private/internal AI endpoints, \
set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable"
)),
e => Error::from(e),
})?;
if target.pinned_addrs().is_empty() {
return Ok(Cow::Borrowed(&HTTP_CLIENT));
}
let client = target
.apply_dns_pinning(ai_http_client_builder())
.build()
.map_err(to_anyhow)?;
Ok(Cow::Owned(client))
}
#[derive(Deserialize, Debug)]
struct AIOAuthResource {
client_id: String,
@@ -303,23 +351,13 @@ async fn get_token_using_oauth(
// Validate the resolved token_url against SSRF rules before issuing the request,
// mirroring the protection applied to base_url in `get_base_url` (same
// ALLOW_PRIVATE_AI_BASE_URLS opt-in). Without this a workspace member could
// point token_url at an internal/metadata address.
if !*windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS {
use windmill_common::ssrf::SsrfValidationError;
windmill_common::ssrf::validate_url_for_ssrf(&resource.token_url)
.await
.map_err(|e| match e {
e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!(
"{e}. If you need to use private/internal AI endpoints, \
set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable"
)),
e => Error::from(e),
})?;
}
// point token_url at an internal/metadata address. The returned client pins
// DNS to the validated address so the connect cannot rebind after the check.
let client = pinned_ai_client_for(&resource.token_url).await?;
let mut params = HashMap::new();
params.insert("grant_type", "client_credentials");
params.insert("scope", "https://cognitiveservices.azure.com/.default");
let response = HTTP_CLIENT
let response = client
.post(resource.token_url)
.form(&params)
.basic_auth(resource.client_id, Some(resource.client_secret))
@@ -420,8 +458,11 @@ fn is_sse_response(headers: &HeaderMap) -> bool {
.unwrap_or(false)
}
fn proxy_request_to_request_builder(proxy_request: ProxyRequest) -> RequestBuilder {
let mut request = HTTP_CLIENT.request(proxy_request.method.clone(), &proxy_request.url);
fn proxy_request_to_request_builder(
client: &Client,
proxy_request: ProxyRequest,
) -> RequestBuilder {
let mut request = client.request(proxy_request.method.clone(), &proxy_request.url);
for (header_name, header_value) in &proxy_request.headers {
request = request.header(header_name.as_str(), header_value.as_str());
}
@@ -546,6 +587,8 @@ async fn global_proxy(
custom_headers: HashMap::new(),
};
let client = pinned_ai_client_for(&credentials.base_url).await?;
if matches!(proxy_mode, ProxyExecutionMode::NativeGoogleAi) {
let proxy_args = ProxyBuildArgs {
method: &method,
@@ -558,8 +601,8 @@ async fn global_proxy(
audit_global_ai_request(&db, &authed).await?;
let response = match ai_path.as_str() {
"chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await,
"models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await,
"chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await,
"models" => handle_google_ai_models_proxy(&client, &proxy_args).await,
_ => Err(Error::BadRequest(format!(
"Unsupported Google AI path: {}",
ai_path
@@ -582,7 +625,7 @@ async fn global_proxy(
body: &body,
credentials: &credentials,
})?;
proxy_request_to_request_builder(proxy_request)
proxy_request_to_request_builder(&client, proxy_request)
}
ProxyExecutionMode::NativeGoogleAi | ProxyExecutionMode::NativeAwsBedrock => {
return Err(Error::BadRequest(format!(
@@ -826,9 +869,11 @@ async fn proxy(
credentials: &credentials,
};
let client = pinned_ai_client_for(&credentials.base_url).await?;
let response = match ai_path.as_str() {
"chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await,
"models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await,
"chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await,
"models" => handle_google_ai_models_proxy(&client, &proxy_args).await,
_ => Err(Error::BadRequest(format!(
"Unsupported Google AI path: {}",
ai_path
@@ -887,7 +932,8 @@ async fn proxy(
body: &body,
credentials: &credentials,
})?;
proxy_request_to_request_builder(proxy_request)
let client = pinned_ai_client_for(&credentials.base_url).await?;
proxy_request_to_request_builder(&client, proxy_request)
}
ProxyExecutionMode::NativeGoogleAi => {
return Err(Error::internal_err(
backend/windmill-api/src/apps.rs
+89 -149
View File
@@ -69,7 +69,7 @@ use windmill_common::{
users::username_to_permissioned_as,
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
Pagination, RunnableKind, StripPath,
strip_json_nul, Pagination, RunnableKind, StripPath,
},
variables::{build_crypt, build_crypt_with_key_suffix, encrypt},
worker::{to_raw_value, CLOUD_HOSTED},
@@ -383,11 +383,7 @@ async fn list_search_apps(
// apps' definitions. `check_scopes` uses ScopeDefinition::includes, where run
// does NOT include read, so it correctly denies such tokens.
check_scopes(&authed, || "apps:read".to_string())?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let mut tx = user_db.begin(&authed).await?;
let allowed = build_scope_path_predicate(&authed, "apps", "read");
@@ -1832,50 +1828,6 @@ fn custom_path_conflict_error(
}
}
/// App values live in a `json` column, which — unlike `jsonb` — accepts the
/// `\u0000` escape. Any later `json`→`jsonb` conversion (a workspace fork's
/// `clone_apps`, search indexing, …) then aborts with "unsupported Unicode
/// escape sequence". Strip genuine NULs so the value is jsonb-safe before it
/// lands in the DB; the usual source is a binary file such as `.DS_Store`
/// accidentally bundled into a raw app's file map. A real NUL is unstorable
/// either way, and frontend code that needs the character writes it as the
/// source escape `\u0000`, which JSON-encodes to `\\u0000` (an escaped
/// backslash — the even-parity case below) and is left untouched.
///
/// Returns `Cow::Borrowed` (no allocation) when the value is already clean.
fn strip_null_chars(raw: &str) -> Cow<'_, str> {
let bytes = raw.as_bytes();
let mut out: Option<String> = None;
let mut copied_to = 0;
let mut search_from = 0;
// A genuine NUL is `\u0000`: a `u0000` introduced by an *odd* run of
// backslashes. An even run (`\\u0000`) is an escaped backslash then the
// literal text "u0000" (common in minified JS regexes) and is preserved.
while let Some(rel) = raw[search_from..].find("u0000") {
let at = search_from + rel;
let mut backslashes = 0;
let mut j = at;
while j > 0 && bytes[j - 1] == b'\\' {
backslashes += 1;
j -= 1;
}
if backslashes % 2 == 1 {
// Drop the escaping backslash + `u0000` — the 6 chars in [at-1, at+5).
let out = out.get_or_insert_with(String::new);
out.push_str(&raw[copied_to..at - 1]);
copied_to = at + 5;
}
search_from = at + 5;
}
match out {
Some(mut out) => {
out.push_str(&raw[copied_to..]);
Cow::Owned(out)
}
None => Cow::Borrowed(raw),
}
}
async fn create_app_internal<'a>(
authed: ApiAuthed,
db: sqlx::Pool<sqlx::Postgres>,
@@ -2021,7 +1973,7 @@ async fn create_app_internal<'a>(
.await?;
// `.get()` keeps the raw text (and thus key order); strip any NUL so the
// `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it.
let value = strip_null_chars(app.value.0.get());
let value = strip_json_nul(app.value.0.get());
if matches!(value, Cow::Owned(_)) {
tracing::warn!(path = %app.path, "stripped NUL character(s) from app value on create");
}
@@ -2606,7 +2558,7 @@ async fn update_app_internal<'a>(
// `.get()` keeps the raw text (and thus key order); strip any NUL so the
// `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it.
let value = strip_null_chars(nvalue.0.get());
let value = strip_json_nul(nvalue.0.get());
if matches!(value, Cow::Owned(_)) {
tracing::warn!(path = %npath, "stripped NUL character(s) from app value on update");
}
@@ -3883,6 +3835,18 @@ async fn get_on_behalf_authed_from_app(
Ok((on_behalf_authed, policy))
}
/// Which identity a deployed `apps_u/*` S3 read runs as.
#[cfg(feature = "parquet")]
enum AppS3ReadIdentity {
/// The gate passed: read with the policy's on-behalf identity (the app author in
/// author-mode, the viewer in viewer-mode).
OnBehalf,
/// The gate did not pass but a logged-in, non-embed viewer is present: read with
/// the viewer's OWN identity so the downstream S3 permission check self-enforces
/// their entitlement (never the author's).
AsViewer(ApiAuthed),
}
#[cfg(feature = "parquet")]
async fn check_if_allowed_to_access_s3_file_from_app(
db: &DB,
@@ -3891,7 +3855,7 @@ async fn check_if_allowed_to_access_s3_file_from_app(
w_id: &str,
path: &str,
policy: &Policy,
) -> Result<()> {
) -> Result<AppS3ReadIdentity> {
let is_app_embed = opt_authed.as_ref().is_some_and(|authed| {
windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref())
});
@@ -3912,7 +3876,7 @@ async fn check_if_allowed_to_access_s3_file_from_app(
&db,
)
.await?;
return Ok(());
return Ok(AppS3ReadIdentity::OnBehalf);
}
}
@@ -3921,24 +3885,25 @@ async fn check_if_allowed_to_access_s3_file_from_app(
// get_workspace_s3_resource_and_check_paths already bounds the read by
// their own perms — no provenance gate (it would over-restrict). Embed
// tokens are excluded (untrusted app JS stays confined below).
Ok(())
} else {
// Author-mode/embed: confine reads to the app's declared keys or files THIS
// app produced, else a viewer could launder the author's S3 perms via an
// arbitrary file_key (confused deputy). Provenance is the un-forgeable
// app-origination marker (`trigger_kind='app'` + `trigger=<this app>`);
// `created_by=<caller>` is ANDed only as a per-viewer isolation filter (it
// can narrow — one viewer can't read another's result — never forge).
let creator = opt_authed
.as_ref()
.map(|authed| authed.username.clone())
.unwrap_or_else(|| "anonymous".to_string());
let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| {
keys.iter()
.any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage)
}) || {
sqlx::query_scalar!(
r#"SELECT EXISTS (
return Ok(AppS3ReadIdentity::OnBehalf);
}
// Author-mode/embed: confine reads to the app's declared keys or files THIS
// app produced, else a viewer could launder the author's S3 perms via an
// arbitrary file_key (confused deputy). Provenance is the un-forgeable
// app-origination marker (`trigger_kind='app'` + `trigger=<this app>`);
// `created_by=<caller>` is ANDed only as a per-viewer isolation filter (it
// can narrow — one viewer can't read another's result — never forge).
let creator = opt_authed
.as_ref()
.map(|authed| authed.username.clone())
.unwrap_or_else(|| "anonymous".to_string());
let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| {
keys.iter()
.any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage)
}) || {
sqlx::query_scalar!(
r#"SELECT EXISTS (
SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.workspace_id = $2
AND c.started_at > now() - interval '3 hours'
@@ -3947,21 +3912,45 @@ async fn check_if_allowed_to_access_s3_file_from_app(
AND j.trigger = $3
AND j.created_by = $4
)"#,
file_query.s3,
w_id,
path,
creator,
)
.fetch_one(db)
.await?
.unwrap_or(false)
};
file_query.s3,
w_id,
path,
creator,
)
.fetch_one(db)
.await?
.unwrap_or(false)
};
if !allowed {
Err(Error::BadRequest("File restricted".to_string()))
} else {
Ok(())
if allowed {
return Ok(AppS3ReadIdentity::OnBehalf);
}
// Gate denied. A viewer whose token is effectively unscoped falls back to reading
// as THEMSELVES: the file is still bounded by their own S3 perms downstream, and
// such a token can already fetch it via `job_helpers/download_s3_file`, so the
// fallback adds zero capability. `is_effectively_unscoped` (the same predicate the
// route-scope middleware uses) is what makes that true: a genuinely scope-restricted
// token (e.g. `apps:read:<path>`) is allowed on `apps_u/*` but REJECTED on
// `job_helpers/*`, so serving it the file here WOULD be a new capability — it stays
// gated. `!is_app_embed` keeps that confinement explicit (embed tokens carry the
// `app_embed` scope, so they are already scope-restricted). Anonymous callers (no
// identity) also have no viewer to fall back to. Only the confused-deputy denial
// reaches the message below.
match opt_authed.as_ref() {
Some(viewer)
if !is_app_embed
&& windmill_api_auth::is_effectively_unscoped(viewer.scopes.as_deref()) =>
{
Ok(AppS3ReadIdentity::AsViewer(viewer.clone()))
}
_ => Err(Error::BadRequest(format!(
"S3 file \"{}\" is not accessible from this app. A deployed app running on \
behalf of its author only serves files it generated, files in its declared \
allowlist, or presigned files. To expose a pre-existing file, sign it \
(signS3Object / sign_s3_object) or set the app's execution mode to \"viewer\".",
file_query.s3
))),
}
}
@@ -4029,7 +4018,7 @@ async fn download_s3_file_from_app(
get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, force_viewer_allowed_s3_keys)
.await?;
check_if_allowed_to_access_s3_file_from_app(
let read_authed = match check_if_allowed_to_access_s3_file_from_app(
&db,
&opt_authed,
&query.file_query,
@@ -4037,10 +4026,14 @@ async fn download_s3_file_from_app(
&path,
&policy,
)
.await?;
.await?
{
AppS3ReadIdentity::OnBehalf => on_behalf_authed,
AppS3ReadIdentity::AsViewer(viewer) => viewer,
};
download_s3_file_internal(
OptJobAuthed { authed: on_behalf_authed, job_id: None },
OptJobAuthed { authed: read_authed, job_id: None },
&db,
None,
&w_id,
@@ -4092,9 +4085,15 @@ async fn app_s3_on_behalf_and_provenance(
}
let (on_behalf_authed, policy) =
get_on_behalf_authed_from_app(db, path, w_id, opt_authed, None).await?;
check_if_allowed_to_access_s3_file_from_app(db, opt_authed, file_query, w_id, path, &policy)
.await?;
Ok(crate::db::OptJobAuthed { authed: on_behalf_authed, job_id: None })
let read_authed = match check_if_allowed_to_access_s3_file_from_app(
db, opt_authed, file_query, w_id, path, &policy,
)
.await?
{
AppS3ReadIdentity::OnBehalf => on_behalf_authed,
AppS3ReadIdentity::AsViewer(viewer) => viewer,
};
Ok(crate::db::OptJobAuthed { authed: read_authed, job_id: None })
}
// The app-scoped display ops carry the app path in the URL and everything else
@@ -4798,62 +4797,3 @@ mod embed_token_tests {
assert!(parse_embed_policy("not json").is_err());
}
}
#[cfg(test)]
mod strip_null_chars_tests {
use super::strip_null_chars;
use std::borrow::Cow;
// Build `{"k":"<n backslashes>u0000"}` without writing the escape literally
// (a real NUL can't live in Rust source). Odd n => the trailing `u0000` is a
// genuine NUL escape; even n => an escaped backslash then the text "u0000".
fn doc(backslashes: usize) -> String {
format!(r#"{{"k":"{}u0000"}}"#, "\\".repeat(backslashes))
}
#[test]
fn strips_genuine_null_escape() {
// 1 backslash: the NUL escape is dropped, the string value becomes "".
assert_eq!(strip_null_chars(&doc(1)).as_ref(), r#"{"k":""}"#);
// 3 backslashes: escaped backslash + NUL -> keep the escaped backslash.
let three = doc(3);
let out = strip_null_chars(&three);
assert_eq!(out.as_ref(), r#"{"k":"\\"}"#);
// Result is now valid, NUL-free JSON (i.e. jsonb-safe).
let v: serde_json::Value = serde_json::from_str(out.as_ref()).unwrap();
assert!(!v["k"].as_str().unwrap().as_bytes().contains(&0u8));
}
#[test]
fn preserves_escaped_backslash_then_literal_u0000() {
// Even runs are the literal text "u0000" (e.g. a minified JS regex char
// class) and must be returned untouched, with no allocation.
for n in [2usize, 4] {
let s = doc(n);
let out = strip_null_chars(&s);
assert_eq!(out.as_ref(), s.as_str());
assert!(matches!(out, Cow::Borrowed(_)), "n={n} should be borrowed");
}
}
#[test]
fn preserves_clean_values() {
// Plain value, and the bare token "u0000" with no preceding backslash.
for s in [r#"{"files":{"/index.tsx":"hello"}}"#, r#"{"k":"u0000"}"#] {
let out = strip_null_chars(s);
assert_eq!(out.as_ref(), s);
assert!(matches!(out, Cow::Borrowed(_)));
}
// The escape for a literal backslash char (`u005c`) then text "u0000":
// the only "u0000" match is preceded by `c` (0 backslashes) -> no NUL.
let s = format!(r#"{{"k":"{}u005cu0000"}}"#, "\\");
assert!(matches!(strip_null_chars(&s), Cow::Borrowed(_)));
}
#[test]
fn strips_multiple_and_preserves_surrounding() {
// Mirrors the .DS_Store case: several NULs interleaved with real text.
let s = format!(r#"{{"a":"x{b}u0000{b}u0000y","b":"ok"}}"#, b = "\\");
assert_eq!(strip_null_chars(&s).as_ref(), r#"{"a":"xy","b":"ok"}"#);
}
}
backend/windmill-api/src/drafts.rs
+3 -111
View File
@@ -19,6 +19,7 @@ use windmill_common::{
error::{Error, Result},
user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX},
users::resolve_username_to_email,
utils::strip_json_nul,
variables::{build_crypt, encrypt},
};
@@ -353,7 +354,7 @@ async fn update_draft(
// `draft.value` is a `json` column, so a U+0000 (NUL) would persist as an
// escape and later make any `->>`/`to_jsonb` extraction raise `22P05`.
// Strip it here so a NUL never reaches the column.
let serialized = strip_json_nul(serialized);
let serialized = strip_json_nul(&serialized);
// Upsert. The conflict check rides on the DO UPDATE WHERE clause —
// when the row is newer than `last_sync`, RETURNING yields nothing.
// `created_at` defaults to `now()` but the migration overrides it ($8)
@@ -371,7 +372,7 @@ async fn update_draft(
email,
path,
kind as UserDraftItemKind,
serialized,
serialized.as_ref(),
req.last_sync,
req.force,
req.created_at,
@@ -521,53 +522,6 @@ async fn migrate_legacy_draft(
}
}
/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to
/// store in the `json`-typed `draft.value` (a NUL there would later make any
/// `->>`/`to_jsonb` extraction raise `22P05`).
///
/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a
/// backslash only ever occurs inside a string, so one backslash-parity-aware
/// pass removes every real NUL escape — covering values and keys alike — while
/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal
/// text `u0000`) intact. O(n) over the bytes with no `serde_json::Value` tree to
/// allocate, and the fast path (no such substring at all) returns the input
/// untouched. The slow path is reached not only by genuinely poisoned values but
/// by any value that legitimately contains `u0000` after a backslash (e.g. script
/// source), so it must stay allocation-light for potentially large drafts.
fn strip_json_nul(serialized: String) -> String {
if !serialized.contains("\\u0000") {
return serialized;
}
let bytes = serialized.as_bytes();
let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] != b'\\' {
out.push(bytes[i]);
i += 1;
continue;
}
// Consume the whole run of backslashes. An even run is N/2 escaped
// backslashes and leaves the next char unescaped; an odd run ends in an
// escaping backslash, so a following `u0000` is a real NUL escape.
let run_start = i;
while i < bytes.len() && bytes[i] == b'\\' {
i += 1;
}
let run = i - run_start;
if run % 2 == 1 && bytes[i..].starts_with(b"u0000") {
// Drop the escaping backslash + `u0000`; keep the leading literal pairs.
out.extend(std::iter::repeat(b'\\').take(run - 1));
i += 5;
} else {
out.extend(std::iter::repeat(b'\\').take(run));
}
}
// Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain
// valid UTF-8 (and valid JSON).
String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8")
}
/// For variable-kind drafts with `variable.is_secret == true`, encrypt
/// `variable.value` with the workspace crypt key and mark it
/// `$encrypted:<base64>` so the secret never persists in plaintext at rest.
@@ -859,65 +813,3 @@ async fn require_can_read_path(
}
Err(Error::NotFound(format!("no draft visible at {path}")))
}
#[cfg(test)]
mod tests {
use super::strip_json_nul;
// Parse the (NUL-free) result so assertions read clearly.
fn parsed(s: String) -> serde_json::Value {
serde_json::from_str(&s).expect("strip_json_nul must return valid JSON")
}
#[test]
fn clean_value_is_returned_byte_for_byte() {
let s = r#"{"summary":"all good","n":1}"#.to_string();
assert_eq!(strip_json_nul(s.clone()), s);
}
#[test]
fn real_nul_in_value_is_stripped() {
let out = strip_json_nul(r#"{"summary":"hi\u0000there"}"#.to_string());
assert!(!out.contains(r"\u0000"));
assert_eq!(parsed(out)["summary"], "hithere");
}
#[test]
fn legit_escaped_backslash_is_a_noop() {
// JSON "a\\u0000b" decodes to the 8-char string a,backslash,u,0,0,0,0,b
// — not a NUL — so the value is already clean and round-trips byte-for-byte.
let s = r#"{"summary":"a\\u0000b"}"#.to_string();
assert_eq!(strip_json_nul(s.clone()), s);
}
#[test]
fn collision_real_and_literal_both_handled() {
// "a" carries a real NUL; "b" carries the literal text backslash-u0000.
// The value walk strips the former and leaves the latter intact — the
// pathological case that needed a fallback in SQL is trivial in Rust.
let v = parsed(strip_json_nul(
r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string(),
));
assert_eq!(v["a"], "xy");
assert_eq!(v["b"], "p\\u0000q");
}
#[test]
fn nested_values_and_keys_are_cleaned() {
let out = strip_json_nul(
r#"{"o":{"k\u0000":["a\u0000b",{"deep\u0000":"v\u0000"}]}}"#.to_string(),
);
assert!(!out.contains(r"\u0000"));
let v = parsed(out);
assert_eq!(v["o"]["k"][0], "ab");
assert_eq!(v["o"]["k"][1]["deep"], "v");
}
#[test]
fn odd_backslash_run_keeps_literal_drops_nul() {
// JSON "a\\\u0000b" is an escaped backslash (kept) immediately followed by
// a real NUL escape (dropped) -> decodes to a,backslash,b.
let v = parsed(strip_json_nul(r#"{"x":"a\\\u0000b"}"#.to_string()));
assert_eq!(v["x"], "a\\b");
}
}
backend/windmill-api/src/hub_publish.rs
+543
View File
@@ -0,0 +1,543 @@
use crate::auth::Tokened;
use crate::db::ApiAuthed;
use crate::HTTP_CLIENT;
use axum::{
extract::{FromRequestParts, Json, Path, Query, RawPathParams},
http::{request::Parts, StatusCode},
response::{IntoResponse, Response},
routing::{get, post},
Router,
};
use serde::{Deserialize, Deserializer, Serialize};
use windmill_common::{
error::{to_anyhow, Error},
utils::require_admin,
HUB_BASE_URL,
};
pub fn workspaced_service() -> Router {
Router::new()
.route("/publish_draft", post(publish_draft))
.route("/scripts", post(publish_script))
.route("/flows", post(publish_flow))
.route("/apps", post(publish_app))
.route("/raw_apps", post(publish_raw_app))
.route("/raw_apps/{id}/embed", post(publish_raw_app_embed))
.route(
"/scripts/{ask_id}/recording",
post(publish_script_recording),
)
.route("/flows/{flow_id}/recording", post(publish_flow_recording))
.route(
"/projects/{slug}/pipeline_recording",
post(publish_pipeline_recording),
)
.route("/resource_types", post(publish_resource_type))
.route("/resources", post(publish_resources))
.route("/triggers", post(publish_triggers))
.route("/migrations", post(publish_migrations))
.route("/projects/{slug}/export", get(get_project_export))
.route("/projects/{slug}/submit", post(submit_project))
.route("/project", get(get_project_by_source))
}
#[derive(Deserialize)]
struct HubScope {
folder: Option<String>,
}
fn validate_folder(folder: &str) -> Result<(), Error> {
let ok = !folder.is_empty()
&& folder.len() <= 255
&& folder
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-');
if ok {
Ok(())
} else {
Err(Error::BadRequest(format!("invalid folder: {folder}")))
}
}
fn source_key(workspace: &str, folder: &str) -> Result<String, Error> {
validate_folder(folder)?;
Ok(format!("{workspace}:{folder}"))
}
fn validate_project_slug(slug: &str) -> Result<(), Error> {
let ok = slug.len() >= 3
&& slug.len() <= 50
&& !slug.starts_with('-')
&& !slug.ends_with('-')
&& slug
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
if ok {
Ok(())
} else {
Err(Error::BadRequest(format!("invalid project slug: {slug}")))
}
}
/// A Hub project slug that is valid by construction: deserialization (from a
/// request body or a path segment) is the only way to obtain one and it runs
/// `validate_project_slug`, so no handler can forward or interpolate an
/// unvalidated slug into a Hub URL.
#[derive(Serialize)]
#[serde(transparent)]
struct ProjectSlug(String);
impl<'de> Deserialize<'de> for ProjectSlug {
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let s = String::deserialize(d)?;
validate_project_slug(&s).map_err(serde::de::Error::custom)?;
Ok(ProjectSlug(s))
}
}
impl std::fmt::Display for ProjectSlug {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
/// The single gate every Hub endpoint goes through: an admin caller, their
/// token (the Hub authenticates it back against this instance's whoami), and
/// the validated `workspace_id:folder` source key scoping ownership Hub-side.
/// Handlers can only reach the Hub via this extractor's methods, so a new
/// endpoint cannot forget the admin check or folder validation.
///
/// A workspace can publish one Hub project per folder. The stable, never-mutated
/// link key is `workspace_id:folder_name` (folder name is the path segment and is
/// never renamed — only display_name changes). `:` is safe: neither workspace ids
/// nor folder names (alphanumeric, underscore, hyphen) contain it.
struct HubPublishCtx {
source_id: Option<String>,
token: String,
}
impl<S> FromRequestParts<S> for HubPublishCtx
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request_parts(
parts: &mut Parts,
state: &S,
) -> std::result::Result<Self, Self::Rejection> {
let authed = ApiAuthed::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let tokened = Tokened::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let params = RawPathParams::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let workspace = params
.iter()
.find(|(k, _)| *k == "workspace_id")
.map(|(_, v)| v.to_owned());
let Query(scope) = Query::<HubScope>::from_request_parts(parts, state)
.await
.map_err(IntoResponse::into_response)?;
let build = || -> Result<HubPublishCtx, Error> {
require_admin(authed.is_admin, &authed.username)?;
let workspace = workspace.ok_or_else(|| {
Error::internal_err(
"hub publish route must be nested under /w/{workspace_id}".to_string(),
)
})?;
let source_id = scope
.folder
.as_deref()
.map(|f| source_key(&workspace, f))
.transpose()?;
Ok(HubPublishCtx { source_id, token: tokened.token })
};
build().map_err(IntoResponse::into_response)
}
}
impl HubPublishCtx {
fn require_source(&self) -> Result<&str, Error> {
self.source_id
.as_deref()
.ok_or_else(|| Error::BadRequest("missing folder query param".to_string()))
}
async fn post<T: Serialize>(
&self,
path: &str,
body: &T,
) -> Result<(StatusCode, String), Error> {
forward_to_hub(path, self.require_source()?, &self.token, body).await
}
async fn get(&self, path: &str) -> Result<(StatusCode, String), Error> {
get_from_hub(path, self.require_source()?, &self.token).await
}
/// GET without requiring a folder scope. Only for reads the Hub allows
/// publicly (e.g. exporting an approved project); the empty source id makes
/// the Hub skip the ownership match.
async fn get_maybe_unscoped(&self, path: &str) -> Result<(StatusCode, String), Error> {
get_from_hub(path, self.source_id.as_deref().unwrap_or(""), &self.token).await
}
}
#[derive(Deserialize, Serialize)]
struct PublishDraftBody {
slug: ProjectSlug,
name: String,
summary: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
readme: Option<String>,
}
async fn publish_draft(
ctx: HubPublishCtx,
Json(body): Json<PublishDraftBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post("/projects", &body).await
}
#[derive(Deserialize, Serialize)]
struct PublishScriptBody {
summary: String,
app: String,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
kind: Option<String>,
content: String,
language: String,
#[serde(skip_serializing_if = "Option::is_none")]
schema: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
lockfile: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
source_path: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_script(
ctx: HubPublishCtx,
Json(body): Json<PublishScriptBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post("/scripts/add", &body).await
}
#[derive(Deserialize, Serialize)]
struct PublishFlowInner {
summary: String,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
value: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
schema: Option<serde_json::Value>,
}
#[derive(Deserialize, Serialize)]
struct PublishFlowBody {
flow: PublishFlowInner,
apps: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
source_path: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_flow(
ctx: HubPublishCtx,
Json(body): Json<PublishFlowBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post("/flows", &body).await
}
#[derive(Deserialize, Serialize)]
struct PublishAppBody {
app: serde_json::Value,
apps: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
summary: String,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
source_path: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_app(
ctx: HubPublishCtx,
Json(body): Json<PublishAppBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post("/apps", &body).await
}
#[derive(Deserialize, Serialize)]
struct PublishRawAppBody {
raw: String,
apps: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
summary: String,
#[serde(skip_serializing_if = "Option::is_none")]
path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
source_path: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_raw_app(
ctx: HubPublishCtx,
Json(body): Json<PublishRawAppBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post("/raw_apps", &body).await
}
#[derive(Deserialize, Serialize)]
struct RawAppEmbedBody {
// No skip_serializing_if: `null` must reach the Hub to clear the embed (unpublish).
external_embed_url: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_raw_app_embed(
ctx: HubPublishCtx,
Path((_workspace, id)): Path<(String, i64)>,
Json(body): Json<RawAppEmbedBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/raw_apps/{}/embed", id), &body).await
}
#[derive(Deserialize, Serialize)]
struct RecordingBody {
#[serde(skip_serializing_if = "Option::is_none")]
recording: Option<serde_json::Value>,
project_slug: ProjectSlug,
}
async fn publish_script_recording(
ctx: HubPublishCtx,
Path((_workspace, ask_id)): Path<(String, i64)>,
Json(body): Json<RecordingBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/scripts/{}/recording", ask_id), &body)
.await
}
async fn publish_flow_recording(
ctx: HubPublishCtx,
Path((_workspace, flow_id)): Path<(String, i64)>,
Json(body): Json<RecordingBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/flows/{}/recording", flow_id), &body)
.await
}
// A data-pipeline recording is scoped to the whole project (a folder cascade),
// not a single Hub item, so the slug comes from the path (validated by
// construction) and only the opaque recording is forwarded.
#[derive(Deserialize, Serialize)]
struct PipelineRecordingBody {
#[serde(skip_serializing_if = "Option::is_none")]
recording: Option<serde_json::Value>,
}
async fn publish_pipeline_recording(
ctx: HubPublishCtx,
Path((_workspace, slug)): Path<(String, ProjectSlug)>,
Json(body): Json<PipelineRecordingBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/projects/{}/pipeline_recording", slug), &body)
.await
}
#[derive(Deserialize, Serialize)]
struct PublishResourceTypeBody {
name: String,
#[serde(skip_serializing_if = "Option::is_none")]
schema: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
project_slug: ProjectSlug,
}
async fn publish_resource_type(
ctx: HubPublishCtx,
Json(body): Json<PublishResourceTypeBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(
&format!("/projects/{}/resource_types", body.project_slug),
&body,
)
.await
}
#[derive(Deserialize, Serialize)]
struct PublishResourceBody {
path: String,
resource_type: String,
}
#[derive(Deserialize, Serialize)]
struct PublishResourcesBody {
resources: Vec<PublishResourceBody>,
project_slug: ProjectSlug,
}
async fn publish_resources(
ctx: HubPublishCtx,
Json(body): Json<PublishResourcesBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/projects/{}/resources", body.project_slug), &body)
.await
}
#[derive(Deserialize, Serialize)]
struct PublishTriggerBody {
path: String,
kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
summary: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
description: Option<String>,
config: serde_json::Value,
#[serde(skip_serializing_if = "Option::is_none")]
script_ask_id: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
flow_id: Option<i64>,
}
#[derive(Deserialize, Serialize)]
struct PublishTriggersBody {
triggers: Vec<PublishTriggerBody>,
project_slug: ProjectSlug,
}
async fn publish_triggers(
ctx: HubPublishCtx,
Json(body): Json<PublishTriggersBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(&format!("/projects/{}/triggers", body.project_slug), &body)
.await
}
// One best-effort data table migration attached to a project (per data table).
#[derive(Deserialize, Serialize)]
struct PublishMigrationBody {
datatable_name: String,
sql: String,
#[serde(default)]
sql_down: String,
enabled: bool,
}
#[derive(Deserialize, Serialize)]
struct PublishMigrationsBody {
migrations: Vec<PublishMigrationBody>,
project_slug: ProjectSlug,
}
async fn publish_migrations(
ctx: HubPublishCtx,
Json(body): Json<PublishMigrationsBody>,
) -> Result<impl IntoResponse, Error> {
ctx.post(
&format!("/projects/{}/migrations", body.project_slug),
&body,
)
.await
}
// Export is owner-scoped only when re-exporting your own draft; approved
// projects are public, so the folder scope is optional here.
async fn get_project_export(
ctx: HubPublishCtx,
Path((_workspace, slug)): Path<(String, ProjectSlug)>,
) -> Result<impl IntoResponse, Error> {
ctx.get_maybe_unscoped(&format!("/projects/{}/export", slug))
.await
}
async fn get_project_by_source(ctx: HubPublishCtx) -> Result<impl IntoResponse, Error> {
ctx.get("/projects/by_source").await
}
async fn submit_project(
ctx: HubPublishCtx,
Path((_workspace, slug)): Path<(String, ProjectSlug)>,
) -> Result<impl IntoResponse, Error> {
ctx.post(
&format!("/projects/{}/submit", slug),
&serde_json::json!({}),
)
.await
}
// The Hub has no auth of its own: it validates bearer tokens by calling this
// instance's /api/users/whoami. Forwarding the caller's own token logs them in
// on the Hub as themselves (account auto-created on first use).
async fn get_from_hub(
path: &str,
source_id: &str,
token: &str,
) -> Result<(StatusCode, String), Error> {
let url = format!("{}{}", **HUB_BASE_URL.load(), path);
let res = HTTP_CLIENT
.get(&url)
.query(&[("source_id", source_id)])
.bearer_auth(token)
.send()
.await
.map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?;
let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
let text = res
.text()
.await
.map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?;
Ok((status, text))
}
async fn forward_to_hub<T: Serialize>(
path: &str,
source_id: &str,
token: &str,
body: &T,
) -> Result<(StatusCode, String), Error> {
let url = format!("{}{}", **HUB_BASE_URL.load(), path);
let mut payload = serde_json::to_value(body).map_err(to_anyhow)?;
let obj = payload
.as_object_mut()
.ok_or_else(|| Error::internal_err("hub publish body must be a JSON object".to_string()))?;
obj.insert(
"source_id".to_string(),
serde_json::Value::String(source_id.to_string()),
);
let res = HTTP_CLIENT
.post(&url)
.bearer_auth(token)
.json(&payload)
.send()
.await
.map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?;
let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
let text = res
.text()
.await
.map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?;
Ok((status, text))
}
backend/windmill-api/src/jobs.rs
+71 -31
View File
@@ -25,7 +25,7 @@ use std::time::Instant;
use tokio::io::AsyncReadExt;
use tower::ServiceBuilder;
use url::Url;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
#[cfg(all(feature = "enterprise", feature = "instance_smtp"))]
use windmill_common::auth::is_super_admin_email;
use windmill_common::auth::TOKEN_PREFIX_LEN;
#[cfg(feature = "run_inline")]
@@ -54,7 +54,7 @@ use windmill_common::workspace_dependencies::{
RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES,
};
use windmill_common::DYNAMIC_INPUT_CACHE;
#[cfg(all(feature = "enterprise", feature = "smtp"))]
#[cfg(all(feature = "enterprise", feature = "instance_smtp"))]
use windmill_common::{email_oss::send_email_html, server::load_smtp_config};
use windmill_object_store::upload_artifact_to_store;
#[cfg(feature = "run_inline")]
@@ -1692,7 +1692,7 @@ impl<'a> GetQuery<'a> {
}
}
#[cfg(all(feature = "smtp", feature = "enterprise"))]
#[cfg(all(feature = "instance_smtp", feature = "enterprise"))]
async fn send_workspace_trigger_failure_email_notification(
db: &DB,
w_id: &str,
@@ -1855,7 +1855,7 @@ struct SendEmail {
error: Value,
}
#[cfg(all(feature = "enterprise", feature = "smtp"))]
#[cfg(all(feature = "enterprise", feature = "instance_smtp"))]
async fn send_email_with_instance_smtp(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -1913,7 +1913,7 @@ async fn send_email_with_instance_smtp(
Ok(Json(resp))
}
#[cfg(not(all(feature = "enterprise", feature = "smtp")))]
#[cfg(not(all(feature = "enterprise", feature = "instance_smtp")))]
async fn send_email_with_instance_smtp(
_authed: ApiAuthed,
Extension(_db): Extension<DB>,
@@ -3192,17 +3192,9 @@ async fn resume_suspended(
}
// Check approval conditions
let approval_conditions = if is_wac {
flow.flow_status
.as_ref()
.and_then(|v| v.get("approval_conditions"))
.and_then(|v| serde_json::from_value::<ApprovalConditions>(v.clone()).ok())
} else {
flow.flow_status
.as_ref()
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
.and_then(|fs| fs.approval_conditions)
};
let approval_conditions = extract_approval_conditions(flow.flow_status.as_ref(), is_wac);
let trigger_email = flow.email.as_deref().unwrap_or("");
if let Some(ref ac) = approval_conditions {
if ac.user_auth_required && opt_authed.is_none() {
@@ -3214,6 +3206,13 @@ async fn resume_suspended(
// If logged in, check authorization rules
if let Some(ref authed) = opt_authed {
// self_approval_disabled applies to owners too (only admins are exempt), so it is
// enforced before the owner shortcut below. A token-only (anonymous) resume is treated as
// capability-based and intentionally not gated here; see resume_suspended_job.
if let Some(ref ac) = approval_conditions {
require_not_self_approval(authed, ac, trigger_email)?;
}
let is_admin = authed.is_admin;
let is_owner = flow
.script_path
@@ -3222,7 +3221,6 @@ async fn resume_suspended(
.unwrap_or(false);
if !is_admin && !is_owner {
let trigger_email = flow.email.as_deref().unwrap_or("");
conditionally_require_authed_user(
Some(authed.clone()),
approval_conditions.clone(),
@@ -3347,10 +3345,11 @@ struct ApprovalInfo {
}
/// Whether `opt_authed` is allowed to approve — and therefore view — this approval step.
/// Mirrors the authorization performed at the resume boundary: workspace admins and owners
/// of the runnable always qualify; otherwise the approval conditions (user_auth_required /
/// user_groups_required / self_approval_disabled) decide. When the step does not require auth,
/// an anonymous (token-only) caller qualifies.
/// Mirrors the authorization performed at the resume boundary: workspace admins always qualify;
/// self_approval_disabled then bars the triggerer even when they own the runnable; otherwise
/// owners qualify and the remaining approval conditions (user_auth_required /
/// user_groups_required) decide. When the step does not require auth, an anonymous (token-only)
/// caller qualifies.
fn can_approve_step(
opt_authed: &Option<ApiAuthed>,
approval_conditions: &Option<ApprovalConditions>,
@@ -3362,6 +3361,12 @@ fn can_approve_step(
if authed.is_admin {
return true;
}
// self_approval_disabled applies to owners too, so it gates the owner shortcut.
if let Some(ref ac) = approval_conditions {
if require_not_self_approval(authed, ac, trigger_email).is_err() {
return false;
}
}
let is_owner = script_path
.map(|p| require_owner_of_path(authed, p).is_ok())
.unwrap_or(false);
@@ -3648,8 +3653,10 @@ async fn resume_suspended_job_internal(
// Get flow info - works for step-level, flow-level, and WAC approval
let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?;
// HMAC secret = full capability. Skip approval_conditions checks.
// Authorization rules are enforced by the new resume_suspended endpoint instead.
// HMAC secret = full capability. Skip approval_conditions checks: possession of the full
// resume URL is the authorization (it is only disclosed to intended approvers, e.g. when a
// step returns it). Identity-based rules, including self_approval_disabled, are enforced by
// the resume_suspended endpoint instead.
let exists = sqlx::query_scalar!(
r#"
@@ -4095,6 +4102,45 @@ pub async fn get_suspended_job_flow(
Ok(Json(SuspendedJobFlow { job: flow, approvers, view_token }).into_response())
}
/// Read the step's approval_conditions from the suspended flow status. For classic flows they
/// live inside the deserialized `FlowStatus`; for workflow-as-code they are a top-level
/// `approval_conditions` key in the status JSON.
fn extract_approval_conditions(
flow_status: Option<&serde_json::Value>,
is_wac: bool,
) -> Option<ApprovalConditions> {
if is_wac {
flow_status
.and_then(|v| v.get("approval_conditions"))
.and_then(|v| serde_json::from_value::<ApprovalConditions>(v.clone()).ok())
} else {
flow_status
.and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
.and_then(|fs| fs.approval_conditions)
}
}
/// The flow's triggerer may not approve their own suspended step when the step sets
/// `self_approval_disabled`. Only admins are exempt: owning the runnable does not grant
/// the right to approve your own run, so this must be enforced at every resume boundary
/// independently of the owner shortcut (which only waives user_auth_required /
/// user_groups_required).
fn require_not_self_approval(
authed: &ApiAuthed,
approval_conditions: &ApprovalConditions,
trigger_email: &str,
) -> error::Result<()> {
if approval_conditions.self_approval_disabled
&& !authed.is_admin
&& authed.email.eq(trigger_email)
{
return Err(Error::PermissionDenied(
"Self-approval is disabled for this flow step".to_string(),
));
}
Ok(())
}
fn conditionally_require_authed_user(
_authed: Option<ApiAuthed>,
approval_conditions_opt: Option<ApprovalConditions>,
@@ -4106,14 +4152,8 @@ fn conditionally_require_authed_user(
let approval_conditions = approval_conditions_opt.unwrap();
// Check self-approval independently of user_auth_required
if approval_conditions.self_approval_disabled {
if let Some(ref authed) = _authed {
if !authed.is_admin && authed.email.eq(_trigger_email) {
return Err(Error::PermissionDenied(
"Self-approval is disabled for this flow step".to_string(),
));
}
}
if let Some(ref authed) = _authed {
require_not_self_approval(authed, &approval_conditions, _trigger_email)?;
}
if approval_conditions.user_auth_required {
backend/windmill-api/src/lib.rs
+2
View File
@@ -94,6 +94,7 @@ mod granular_acls;
mod group_history;
mod groups;
mod health;
mod hub_publish;
#[cfg(feature = "private")]
pub mod indexer_ee;
mod indexer_oss;
@@ -656,6 +657,7 @@ pub async fn run_server(
.nest("/volumes", volumes_oss::workspaced_service())
.nest("/workers", windmill_api_workers::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/hub", hub_publish::workspaced_service())
.nest(
"/data_metrics",
windmill_api_workspaces::data_metrics::workspaced_service(),
backend/windmill-api/src/live_migrations.rs
+24
View File
@@ -19,6 +19,30 @@ pub async fn custom_migrations(migrator: &mut CustomMigrator) -> Result<(), Erro
tracing::error!("Could not apply flow versioning fix migration: {err:#}");
}
if let Err(err) = normalize_custom_instance_user_attributes(migrator).await {
tracing::error!("Could not normalize custom_instance_user attributes: {err:#}");
}
Ok(())
}
// Converged on every boot, not once: the one-shot migration swallows errors (it must not
// abort startup without superuser), and an older instance sharing the cluster can re-add
// the attribute. REPLICATION belongs only on custom_instance_replication_user.
async fn normalize_custom_instance_user_attributes(
migrator: &mut CustomMigrator,
) -> Result<(), Error> {
let has_replication = sqlx::query_scalar::<_, bool>(
"SELECT rolreplication FROM pg_roles WHERE rolname = 'custom_instance_user'",
)
.fetch_optional(migrator.connection())
.await?;
if has_replication == Some(true) {
sqlx::query("ALTER ROLE custom_instance_user NOREPLICATION")
.execute(migrator.connection())
.await?;
tracing::info!("Normalized custom_instance_user attributes");
}
Ok(())
}
backend/windmill-common/Cargo.toml
+1
View File
@@ -38,6 +38,7 @@ anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_yml.workspace = true
memchr.workspace = true
erased-serde = "0.4"
chrono.workspace = true
chrono-tz.workspace = true
backend/windmill-common/src/assets.rs
+6 -9
View File
@@ -567,11 +567,9 @@ mod trigger_ref_roundtrip_tests {
// `trigger_spec_to_row` rebuilds a stored ref as `s3://<path>`, and
// `parse_asset_trigger_ref` parses it back. The two must be inverse for
// every S3 URI form, or a consumer's `// on` trigger lands on a different
// graph node than the producer's inferred write. Because `parse_asset_syntax`
// strips ALL leading slashes, a canonical path never starts with `/`, so the
// naive `prefix + path` rebuild round-trips — including the `S3Object(s3="/x")`
// quad-slash case that previously desynced (path `/x` rebuilt to `s3:///x`,
// which re-parsed to `x`).
// graph node than the producer's inferred write. `parse_asset_syntax`
// keeps the URI suffix verbatim (a default-storage path starts with `/`),
// so the naive `prefix + path` rebuild round-trips for every form.
fn roundtrip(uri: &str) -> String {
let (pkind, path) = parse_asset_syntax(uri, false).expect("parse uri");
assert_eq!(pkind, PAssetKind::S3Object);
@@ -590,11 +588,10 @@ mod trigger_ref_roundtrip_tests {
#[test]
fn s3_trigger_ref_roundtrips_for_every_uri_form() {
assert_eq!(roundtrip("s3:///exports/x"), "exports/x"); // SDK default storage
assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // DuckDB / bare
assert_eq!(roundtrip("s3:///exports/x"), "/exports/x"); // SDK default storage
assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // named storage `exports`
assert_eq!(roundtrip("s3://mybucket/exports/x"), "mybucket/exports/x"); // explicit
assert_eq!(roundtrip("s3:////x"), "x"); // S3Object(s3="/x") quad-slash
assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "y=2024/f.parquet"); // Hive
assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "/y=2024/f.parquet"); // Hive
}
}
backend/windmill-common/src/ee_oss.rs
+5
View File
@@ -83,6 +83,11 @@ pub async fn enforce_offline_caps(_db: &DB) -> anyhow::Result<Option<OfflineCapS
Ok(None)
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn alert_on_online_license_expired(_db: &DB) {
// Implementation is not open source
}
#[cfg(not(feature = "private"))]
#[derive(PartialEq, Eq)]
pub enum LicensePlan {
backend/windmill-common/src/global_settings.rs
+7
View File
@@ -165,6 +165,11 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[
INSTANCE_EVENTS_WEBHOOK_SETTING,
OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING,
// Custom-instance DB credentials: `custom_instance_pg_databases` holds `user_pwd`,
// `custom_instance_replication_pwd` holds the REPLICATION-role password. Agent workers
// resolve datatable connections through the dedicated datatable endpoints, never these.
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
];
/// Whether an agent worker may read the given global setting over HTTP.
@@ -381,6 +386,8 @@ mod tests {
INSTANCE_EVENTS_WEBHOOK_SETTING,
OTEL_SETTING,
OTEL_TRACING_PROXY_SETTING,
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
] {
assert!(
!is_setting_readable_by_agent_worker(key),
backend/windmill-common/src/instance_config.rs
+67 -32
View File
@@ -776,7 +776,10 @@ pub enum DucklakeCatalogResourceType {
// Custom instance PG databases
// ---------------------------------------------------------------------------
/// Custom PostgreSQL databases managed by the instance.
/// Custom PostgreSQL databases managed by the instance. `user_pwd` is operator-configurable
/// (resolved from a Kubernetes secretKeyRef by the EE operator); `databases` is runtime
/// setup status. The replication-role password lives in a separate hidden setting
/// (`custom_instance_replication_pwd`), never in this operator-facing config row.
#[derive(Deserialize, Serialize, Clone, Debug, Default)]
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
pub struct CustomInstancePgDatabases {
@@ -945,6 +948,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
"ducklake_user_pg_pwd",
"ducklake_settings",
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
"uid",
"rsa_keys",
"jwt_secret",
@@ -966,6 +970,10 @@ pub const HIDDEN_SETTINGS: &[&str] = &[
// every bulk InstanceSettings save via `GlobalSettings::extra`. Hiding it
// on read + rejecting it in `diff_global_settings` breaks that loop.
"worker_configs",
// Auto-generated password for the REPLICATION role used by postgres triggers.
// Server-only (written by setup/refresh via direct SQL), never operator-authored —
// hidden so the config machinery can't read, rewrite, or drop it.
"custom_instance_replication_pwd",
];
/// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
@@ -976,6 +984,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[
"hub_api_secret",
"license_key",
"ducklake_user_pg_pwd",
"custom_instance_replication_pwd",
"pip_index_url",
"pip_extra_index_url",
"npm_config_registry",
@@ -1000,6 +1009,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
"object_store_cache_config",
&["secret_key", "serviceAccountKey"],
),
("custom_instance_pg_databases", &["user_pwd"]),
];
fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
@@ -1024,10 +1034,7 @@ fn mask_nested_sensitive(key: &str, value: &serde_json::Value) -> serde_json::Va
}
}
// Settings that are maps-of-objects where each child has a sensitive sub-field.
const NESTED_MAP_SENSITIVE: &[(&str, &str)] = &[
("oauths", "secret"),
("custom_instance_pg_databases", "user_pwd"),
];
const NESTED_MAP_SENSITIVE: &[(&str, &str)] = &[("oauths", "secret")];
for &(parent_key, child_field) in NESTED_MAP_SENSITIVE {
if key == parent_key {
if let serde_json::Value::Object(entries) = value {
@@ -1142,14 +1149,13 @@ pub fn diff_global_settings(
let mut previous_values = BTreeMap::new();
let mut unchanged_count: usize = 0;
for (key, desired_value) in desired {
// `worker_configs` is a legacy ghost: worker configs belong in the
// `config` table with a `worker__` prefix. If a client PUT carries a
// top-level `worker_configs` key (it flattens into
// `GlobalSettings::extra` on deserialize), drop it here instead of
// letting it resurrect a stale `global_settings` row.
if key == "worker_configs" {
// Hidden settings are server-managed and never driven by config: they are
// filtered out on read (`from_db`) and must be ignored on write too, so a client
// PUT that flattened one into `GlobalSettings::extra` can't resurrect or clobber
// the row (e.g. `worker_configs`, or the custom-instance credentials/status).
if HIDDEN_SETTINGS.contains(&key.as_str()) {
tracing::warn!(
"Ignoring 'worker_configs' in global_settings diff: worker configs must be written to the config table (worker__ prefix), not global_settings"
"Ignoring hidden setting '{key}' in global_settings diff (server-managed, not configurable)"
);
continue;
}
@@ -2367,34 +2373,63 @@ mod tests {
}
#[test]
fn custom_instance_pg_databases_roundtrips() {
fn custom_instance_replication_pwd_is_isolated_from_config() {
// The replication-role password is server-only: written by setup/refresh via direct
// SQL, never operator-authored. It must stay out of the declarative config surface
// (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it.
assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd"));
// A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
let mut desired = BTreeMap::new();
desired.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("attacker-set"),
);
let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
assert!(
diff.upserts.is_empty(),
"hidden setting must not be upserted"
);
// A current value is never deleted by a Replace that omits it.
let mut current = BTreeMap::new();
current.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("live"),
);
let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
assert!(
!diff
.deletes
.contains(&"custom_instance_replication_pwd".to_string()),
"hidden setting must not be deleted"
);
}
#[test]
fn custom_instance_pg_databases_roundtrips_and_redacts_user_pwd() {
// user_pwd stays operator-configurable (EE secretKeyRef); databases is runtime status.
let json = r#"{
"user_pwd": "secret123",
"databases": {
"mydb": {
"logs": {
"super_admin": "OK",
"database_credentials": "OK",
"valid_dbname": "OK",
"created_database": "OK",
"db_connect": "OK",
"grant_permissions": "OK"
},
"success": true,
"tag": "production"
}
}
"databases": { "mydb": { "success": true, "tag": "production" } }
}"#;
let pg: CustomInstancePgDatabases = serde_json::from_str(json).unwrap();
assert_eq!(
pg.user_pwd.as_ref().and_then(|v| v.as_literal()),
Some("secret123")
);
let db = &pg.databases["mydb"];
assert!(db.success);
assert_eq!(db.tag.as_deref(), Some("production"));
assert_eq!(db.logs.super_admin, "OK");
assert_eq!(db.logs.grant_permissions, "OK");
assert!(pg.databases["mydb"].success);
let out = format_setting_value(
"custom_instance_pg_databases",
&serde_json::json!({ "user_pwd": "user-plaintext-password" }),
);
assert!(
!out.contains("user-plaintext-password"),
"user_pwd leaked: {out}"
);
}
#[test]
backend/windmill-common/src/query_builders.rs
+12 -10
View File
@@ -4835,6 +4835,10 @@ fn pg_action_to_string(action: &str) -> String {
pub async fn pg_get_full_schema(
client: &tokio_postgres::Client,
) -> Result<FullDatabaseSchema, String> {
// Primary-key and default-value info are joined in (a table has at most one
// primary-key constraint, so `pkc` stays 1:1) rather than fetched via
// per-column correlated subqueries — on large catalogs those subqueries run
// once per column and make the introspection time out.
let column_rows = client
.query(
"SELECT
@@ -4842,19 +4846,17 @@ pub async fn pg_get_full_schema(
c.relname AS table_name,
a.attname AS column_name,
pg_catalog.format_type(a.atttypid, a.atttypmod) AS datatype,
(SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
FROM pg_catalog.pg_attrdef d
WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default_value,
CASE a.attnotnull WHEN false THEN true ELSE false END AS nullable,
EXISTS (
SELECT 1 FROM pg_catalog.pg_index i
WHERE i.indrelid = c.oid AND i.indisprimary AND a.attnum = ANY(i.indkey)
) AS is_primary_key,
(SELECT con.conname FROM pg_catalog.pg_constraint con
WHERE con.conrelid = c.oid AND con.contype = 'p' LIMIT 1) AS pk_constraint_name
substring(pg_catalog.pg_get_expr(ad.adbin, ad.adrelid, true) for 128) AS default_value,
NOT a.attnotnull AS nullable,
COALESCE(pkc.conkey @> ARRAY[a.attnum], false) AS is_primary_key,
pkc.conname AS pk_constraint_name
FROM pg_catalog.pg_attribute a
JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid
LEFT JOIN pg_catalog.pg_attrdef ad
ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum AND a.atthasdef
LEFT JOIN pg_catalog.pg_constraint pkc
ON pkc.conrelid = c.oid AND pkc.contype = 'p'
WHERE c.relkind = 'r'
AND a.attnum > 0
AND NOT a.attisdropped
backend/windmill-common/src/ssrf.rs
+113 -13
View File
@@ -1,4 +1,4 @@
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use crate::error::Error;
@@ -58,6 +58,63 @@ impl std::fmt::Display for SsrfValidationError {
// `anyhow::Result` (e.g. the EE SAML metadata loader).
impl std::error::Error for SsrfValidationError {}
/// A URL that passed SSRF validation, carrying the exact addresses its host
/// resolved to so the eventual connect targets the SAME address that was
/// checked.
///
/// Validation resolves the host once and verifies every address is public; it
/// then hands those addresses back instead of discarding them. Callers pin them
/// onto their client — [`apply_dns_pinning`](ValidatedTarget::apply_dns_pinning)
/// for reqwest, or [`pinned_addrs`](ValidatedTarget::pinned_addrs) for a raw TCP
/// connect — so a DNS rebinder cannot answer a public IP at check-time and an
/// internal one (e.g. 169.254.169.254) at connect-time. Without pinning the
/// check and the connect resolve independently and the guard is a TOCTOU no-op.
///
/// `addrs` is empty when the host was an IP literal (there is nothing to rebind)
/// or when an `ALLOW_PRIVATE_*` override skipped resolution entirely; pinning is
/// then a no-op and the caller connects normally.
///
/// Limitation: pinning governs only *direct* connections. When a deployment
/// configures an outbound egress proxy (`HTTP_PROXY`/`HTTPS_PROXY`), the proxy
/// resolves the target host itself and the pin does not reach it — a property of
/// proxy-based egress shared by every app-side SSRF guard, not specific to this
/// one. The public/private pre-check still runs; closing the proxy hop would
/// require the proxy to resolve, which it owns.
#[derive(Debug, Clone)]
pub struct ValidatedTarget {
/// The URL host, exactly as reqwest keys its DNS override on.
pub host: String,
/// Public addresses the host resolved to, to pin at connect time.
pub addrs: Vec<SocketAddr>,
}
impl ValidatedTarget {
/// A target with nothing to pin: an IP-literal host (no rebinding possible)
/// or a host whose SSRF check was skipped by an `ALLOW_PRIVATE_*` override.
fn unpinned(host: &str) -> Self {
ValidatedTarget { host: host.to_string(), addrs: Vec::new() }
}
/// Pin the validated addresses onto a reqwest client builder so connect-time
/// resolution cannot diverge from what was checked. No-op when there is
/// nothing to pin (IP-literal host, or a skipped `ALLOW_PRIVATE_*` check).
pub fn apply_dns_pinning(&self, builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder {
if self.addrs.is_empty() {
builder
} else {
builder.resolve_to_addrs(&self.host, &self.addrs)
}
}
/// The validated addresses to connect to, for callers that pin by opening
/// the socket themselves (e.g. the WebSocket trigger's raw TCP connect)
/// rather than through reqwest. Empty means "nothing to pin, connect
/// normally".
pub fn pinned_addrs(&self) -> &[SocketAddr] {
&self.addrs
}
}
impl From<SsrfValidationError> for Error {
fn from(e: SsrfValidationError) -> Self {
Error::BadRequest(e.to_string())
@@ -69,8 +126,13 @@ impl From<SsrfValidationError> for Error {
/// Checks:
/// 1. Scheme must be http or https
/// 2. Host must be present and not a private/loopback/link-local IP
/// 3. DNS resolution is checked to prevent DNS rebinding to internal IPs
pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError> {
/// 3. The host is resolved and every address verified public
///
/// Returns the resolved addresses as a [`ValidatedTarget`] so the caller can pin
/// them onto the client that actually connects. Validating here and re-resolving
/// at connect time is a TOCTOU no-op against a DNS rebinder — the check only
/// closes the hole if the connect targets the SAME address this resolved.
pub async fn validate_url_for_ssrf(url: &str) -> Result<ValidatedTarget, SsrfValidationError> {
let parsed =
url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?;
@@ -85,12 +147,13 @@ pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError>
// 2. Host check
let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
// 3. If the host is an IP literal, check it directly
// 3. If the host is an IP literal, check it directly. There is nothing to
// rebind (reqwest connects straight to the literal), so no addresses to pin.
if let Ok(ip) = host.parse::<IpAddr>() {
if is_private_ip(&ip) {
return Err(SsrfValidationError::Private { resolved: false });
}
return Ok(());
return Ok(ValidatedTarget::unpinned(host));
}
// 4. DNS resolution check — resolve the hostname and verify all IPs are public
@@ -117,7 +180,7 @@ pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError>
}
}
Ok(())
Ok(ValidatedTarget { host: host.to_string(), addrs })
}
pub fn allow_private_mcp_server_urls() -> bool {
@@ -132,7 +195,7 @@ pub fn allow_private_saml_metadata_urls() -> bool {
.is_some_and(|v| v == "true" || v == "1")
}
pub async fn validate_saml_metadata_url(url: &str) -> Result<(), SsrfValidationError> {
pub async fn validate_saml_metadata_url(url: &str) -> Result<ValidatedTarget, SsrfValidationError> {
let parsed =
url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?;
@@ -141,16 +204,16 @@ pub async fn validate_saml_metadata_url(url: &str) -> Result<(), SsrfValidationE
scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())),
}
parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
if allow_private_saml_metadata_urls() {
return Ok(());
return Ok(ValidatedTarget::unpinned(host));
}
validate_url_for_ssrf(url).await
}
pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationError> {
pub async fn validate_mcp_server_url(url: &str) -> Result<ValidatedTarget, SsrfValidationError> {
let parsed =
url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?;
@@ -159,16 +222,24 @@ pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationErro
scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())),
}
parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?;
if allow_private_mcp_server_urls() {
return Ok(());
return Ok(ValidatedTarget::unpinned(host));
}
validate_url_for_ssrf(url).await
}
pub async fn validate_mcp_server_url_for_bad_request(url: &str, label: &str) -> Result<(), Error> {
/// Validate an MCP-related URL and return the [`ValidatedTarget`] so the caller
/// can pin the connect: the OAuth registration/discovery/token requests carry
/// secrets, so they must target the validated address (see
/// `windmill_mcp::oauth::no_redirect_http_client_pinned`). Callers that only
/// pre-validate (no adjacent connect) can discard the target.
pub async fn validate_mcp_server_url_for_bad_request(
url: &str,
label: &str,
) -> Result<ValidatedTarget, Error> {
validate_mcp_server_url(url).await.map_err(|e| {
Error::BadRequest(format!(
"{label} is not allowed: {}",
@@ -331,6 +402,35 @@ mod tests {
assert!(validate_url_for_ssrf("https://google.com").await.is_ok());
}
/// An IP-literal host has nothing to rebind — reqwest connects straight to
/// the literal — so the target pins no addresses.
#[tokio::test]
async fn validate_url_ip_literal_pins_nothing() {
let target = validate_url_for_ssrf("http://8.8.8.8:1234/x")
.await
.unwrap();
assert_eq!(target.host, "8.8.8.8");
assert!(target.pinned_addrs().is_empty());
}
/// Regression for the DNS-rebinding TOCTOU: the guard must surface the exact
/// public addresses it validated so the caller can pin the connect to the
/// SAME address. If
/// this returned nothing, the connect would re-resolve and a rebinder could
/// swap in an internal IP after the check.
#[tokio::test]
async fn validate_url_surfaces_resolved_addrs_for_pinning() {
let target = validate_url_for_ssrf("https://google.com").await.unwrap();
assert_eq!(target.host, "google.com");
assert!(!target.pinned_addrs().is_empty());
assert!(target
.pinned_addrs()
.iter()
.all(|a| !is_private_ip(&a.ip())));
// The pin applies cleanly onto a reqwest builder.
let _ = target.apply_dns_pinning(reqwest::ClientBuilder::new());
}
/// Regression for #9171: a malformed base URL (missing scheme) must report
/// `InvalidUrl`/`DisallowedScheme`, not `Private` — only `Private` gets the
/// "set ALLOW_PRIVATE_AI_BASE_URLS" hint, which is misleading for a typo'd
backend/windmill-common/src/utils.rs
+229
View File
@@ -563,6 +563,20 @@ pub async fn report_critical_error(
}
}
/// Route a workspace-level failure to the instance critical alert channels without
/// recording an `alerts` row: job failures are workspace noise and would otherwise flood
/// the instance-wide feed superadmins triage. The channels belong to the instance operator,
/// who on cloud is not the workspace owner, hence the hard stop there. Callers own the
/// per-workspace opt-in.
pub async fn send_workspace_error_to_instance_channels(_error_message: String, _db: &DB) -> () {
if *CLOUD_HOSTED {
return;
}
#[cfg(feature = "enterprise")]
send_critical_alert(_error_message, _db, CriticalAlertKind::CriticalError, None).await;
}
pub async fn report_recovered_critical_error(
message: String,
_db: DB,
@@ -1044,6 +1058,92 @@ pub async fn get_custom_pg_instance_password(db: &DB) -> Result<String> {
)
}
const REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL: &str = r#"
DO $$
DECLARE
pwd text;
BEGIN
SELECT gen_random_uuid()::text INTO pwd;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_replication_user') THEN
EXECUTE format('ALTER USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd);
ELSE
EXECUTE format('CREATE USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd);
END IF;
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN
GRANT custom_instance_user TO custom_instance_replication_user;
ALTER ROLE custom_instance_user NOREPLICATION;
END IF;
INSERT INTO global_settings (name, value)
VALUES ('custom_instance_replication_pwd', to_jsonb(pwd::text))
ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value;
END
$$;
"#;
const REPLICATION_PWD_READ_SQL: &str =
"SELECT value #>> '{}' FROM global_settings WHERE name = 'custom_instance_replication_pwd'";
/// (Re)create `custom_instance_replication_user` with a fresh password. This role is
/// used by postgres trigger connections on custom-instance datatables; membership in
/// `custom_instance_user` lets it manage publications on the datatable tables.
///
/// Authorization: rotates a stored database credential and performs no authorization
/// itself — callers MUST restrict this to superadmin or internal server paths.
pub async fn refresh_custom_instance_replication_user_pwd(db: &DB) -> Result<()> {
sqlx::query(REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL)
.execute(db)
.await?;
Ok(())
}
/// Authorization: returns a stored database credential and performs no authorization
/// itself — callers MUST restrict this to superadmin or internal server paths (mirrors
/// [`get_custom_pg_instance_password`]).
pub async fn get_custom_pg_instance_replication_password(db: &DB) -> Result<String> {
// Fast path: already provisioned by the migration.
if let Some(pwd) = sqlx::query_scalar::<_, Option<String>>(REPLICATION_PWD_READ_SQL)
.fetch_optional(db)
.await?
.flatten()
{
return Ok(pwd);
}
// Self-heal when the role-creating migration was swallowed. The advisory lock + re-check
// serialize concurrent workers: otherwise two callers both rotate, and the second
// rotation invalidates the password the first already returned. Rotating and reading in
// one locked transaction keeps the decision atomic.
let mut tx = db.begin().await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtext('custom_instance_replication_pwd'))")
.execute(&mut *tx)
.await?;
if let Some(pwd) = sqlx::query_scalar::<_, Option<String>>(REPLICATION_PWD_READ_SQL)
.fetch_optional(&mut *tx)
.await?
.flatten()
{
tx.commit().await?;
return Ok(pwd);
}
sqlx::query(REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL)
.execute(&mut *tx)
.await?;
let pwd = sqlx::query_scalar::<_, Option<String>>(REPLICATION_PWD_READ_SQL)
.fetch_optional(&mut *tx)
.await?
.flatten()
.ok_or_else(|| {
Error::BadRequest(
"Custom instance replication user password not found, did you run migrations ?"
.to_string(),
)
})?;
tx.commit().await?;
Ok(pwd)
}
/// Convert a JSON string to a `Box<RawValue>` without validation.
///
/// # Safety
@@ -1129,9 +1229,138 @@ pub fn merge_nested_raw_values_to_array<
serde_json::value::RawValue::from_string(result).unwrap()
}
/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to
/// store in a `jsonb` column, which rejects the `\u0000` escape with 22P05
/// ("unsupported Unicode escape sequence"). A `json`-typed column accepts the
/// escape but propagates the same failure to any later `->>`/`to_jsonb`/`json`→
/// `jsonb` conversion.
///
/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a
/// backslash only ever occurs inside a string, so one backslash-parity-aware
/// pass removes every real NUL escape — covering values and keys alike — while
/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal
/// text `u0000`, common in minified JS regexes) intact. O(n) over the bytes with
/// no `serde_json::Value` tree to allocate, and the fast path (no such substring
/// at all) returns the input borrowed and untouched. The slow path is reached
/// not only by genuinely poisoned values but by any value that legitimately
/// contains `u0000` after a backslash (e.g. script source), so it must stay
/// allocation-light for potentially large documents.
pub fn strip_json_nul(serialized: &str) -> Cow<'_, str> {
// SIMD substring scan (several times faster than `str::contains`'s Two-Way)
// for the guard, since this runs on every completed job's serialized result.
if memchr::memmem::find(serialized.as_bytes(), b"\\u0000").is_none() {
return Cow::Borrowed(serialized);
}
let bytes = serialized.as_bytes();
let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
let mut i = 0;
// The substring guard above is satisfied by legitimate `\\u0000` too, so only
// an odd-parity NUL escape actually drops bytes. Borrow back out when nothing
// was stripped, so `Cow::Owned` reliably means "a NUL was removed" — callers
// (e.g. apps.rs) key a warning on that.
let mut stripped = false;
while i < bytes.len() {
if bytes[i] != b'\\' {
out.push(bytes[i]);
i += 1;
continue;
}
// Consume the whole run of backslashes. An even run is N/2 escaped
// backslashes and leaves the next char unescaped; an odd run ends in an
// escaping backslash, so a following `u0000` is a real NUL escape.
let run_start = i;
while i < bytes.len() && bytes[i] == b'\\' {
i += 1;
}
let run = i - run_start;
if run % 2 == 1 && bytes[i..].starts_with(b"u0000") {
// Drop the escaping backslash + `u0000`; keep the leading literal pairs.
out.extend(std::iter::repeat(b'\\').take(run - 1));
i += 5;
stripped = true;
} else {
out.extend(std::iter::repeat(b'\\').take(run));
}
}
if !stripped {
return Cow::Borrowed(serialized);
}
// Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain
// valid UTF-8 (and valid JSON).
Cow::Owned(String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8"))
}
#[cfg(test)]
mod tests {
use super::*;
// The 6-char JSON escape for U+0000: backslash + "u0000". Written via an
// escaped backslash so no literal NUL byte ever appears in this source.
const NUL_ESC: &str = "\\u0000";
// Parse the (NUL-free) result so assertions read clearly.
fn parsed(s: &str) -> serde_json::Value {
serde_json::from_str(s).expect("strip_json_nul must return valid JSON")
}
#[test]
fn strip_json_nul_clean_value_is_borrowed_byte_for_byte() {
let s = r#"{"summary":"all good","n":1}"#;
let out = strip_json_nul(s);
assert!(matches!(out, Cow::Borrowed(_)));
assert_eq!(out, s);
}
#[test]
fn strip_json_nul_real_nul_in_value_is_stripped() {
let input = format!(r#"{{"summary":"hi{NUL_ESC}there"}}"#);
let out = strip_json_nul(&input);
assert!(!out.contains(NUL_ESC));
assert_eq!(parsed(&out)["summary"], "hithere");
}
#[test]
fn strip_json_nul_legit_escaped_backslash_is_a_noop() {
// JSON "a\\u0000b" decodes to a,backslash,u,0,0,0,0,b - not a NUL - so
// the value is already clean and round-trips byte-for-byte. It hits the
// slow path (the substring is present) but strips nothing, so it must
// still return Cow::Borrowed - callers key a "stripped NUL" warning on
// the Owned variant.
let s = r#"{"summary":"a\\u0000b"}"#;
let out = strip_json_nul(s);
assert!(matches!(out, Cow::Borrowed(_)));
assert_eq!(out, s);
}
#[test]
fn strip_json_nul_collision_real_and_literal_both_handled() {
// "a" carries a real NUL escape; "b" carries the literal text backslash-u0000.
let v = parsed(&strip_json_nul(&format!(
r#"{{"a":"x{NUL_ESC}y","b":"p\\u0000q"}}"#
)));
assert_eq!(v["a"], "xy");
assert_eq!(v["b"], "p\\u0000q");
}
#[test]
fn strip_json_nul_nested_values_and_keys_are_cleaned() {
let input =
format!(r#"{{"o":{{"k{NUL_ESC}":["a{NUL_ESC}b",{{"deep{NUL_ESC}":"v{NUL_ESC}"}}]}}}}"#);
let out = strip_json_nul(&input);
assert!(!out.contains(NUL_ESC));
let v = parsed(&out);
assert_eq!(v["o"]["k"][0], "ab");
assert_eq!(v["o"]["k"][1]["deep"], "v");
}
#[test]
fn strip_json_nul_odd_backslash_run_keeps_literal_drops_nul() {
// JSON "a\\ b" is an escaped backslash (kept) immediately followed
// by a real NUL escape (dropped) -> decodes to a,backslash,b.
let v = parsed(&strip_json_nul(&format!(r#"{{"x":"a\\{NUL_ESC}b"}}"#)));
assert_eq!(v["x"], "a\\b");
}
#[test]
fn test_build_arg_str() {
let r = build_arg_str(
backend/windmill-common/src/workspaces.rs
+47 -5
View File
@@ -9,7 +9,7 @@ use crate::{
error::{self, to_anyhow, Error, Result},
get_database_url,
secret_backend::{get_secret_value, is_external_stored_value},
utils::get_custom_pg_instance_password,
utils::{get_custom_pg_instance_password, get_custom_pg_instance_replication_password},
variables::{build_crypt, decrypt},
PgDatabase, DB,
};
@@ -167,7 +167,7 @@ pub enum ObjectType {
DatatableMigration,
}
pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28790/sync-script-to-git-repo-windmill";
pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28796/sync-script-to-git-repo-windmill";
/// Hub script that applies a repository's state back into a workspace
/// (the repo → Windmill / "pull" direction). Same script the UI runs from
@@ -175,7 +175,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28790/sync-script-to-git-repo
/// ignores the slug, so the slug is kept free of characters that would be
/// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened
/// reverse proxies reject as double-encoding when the client re-encodes it).
pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28789/git-sync-init-repository-windmill";
pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28795/git-sync-init-repository-windmill";
/// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a
/// fork of another workspace.
@@ -1118,13 +1118,40 @@ pub async fn datatable_shared_resource(
db: &DB,
w_id: &str,
datatable: &DataTable,
) -> Result<serde_json::Value> {
datatable_shared_resource_inner(db, w_id, datatable, false).await
}
/// Same as [`datatable_shared_resource`] but for postgres trigger connections:
/// custom-instance datatables resolve to `custom_instance_replication_user`
/// rather than `custom_instance_user`. BYO-postgres datatables resolve to the
/// user's own resource unchanged; configuring it for replication there is the
/// user's responsibility.
pub async fn datatable_shared_replication_resource(
db: &DB,
w_id: &str,
datatable: &DataTable,
) -> Result<serde_json::Value> {
datatable_shared_resource_inner(db, w_id, datatable, true).await
}
async fn datatable_shared_resource_inner(
db: &DB,
w_id: &str,
datatable: &DataTable,
replication: bool,
) -> Result<serde_json::Value> {
let db_resource = if datatable.database.resource_type == DataTableCatalogResourceType::Instance
{
let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
pg_creds.dbname = datatable.database.resource_path.clone();
pg_creds.user = Some("custom_instance_user".to_string());
pg_creds.password = Some(get_custom_pg_instance_password(&db).await?);
if replication {
pg_creds.user = Some("custom_instance_replication_user".to_string());
pg_creds.password = Some(get_custom_pg_instance_replication_password(&db).await?);
} else {
pg_creds.user = Some("custom_instance_user".to_string());
pg_creds.password = Some(get_custom_pg_instance_password(&db).await?);
}
serde_json::to_value(&pg_creds)
.map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?
} else {
@@ -1148,6 +1175,21 @@ pub async fn get_datatable_resource_from_db_unchecked(
datatable_shared_resource(db, w_id, &datatable).await
}
/// Same as [`get_datatable_resource_from_db_unchecked`] but resolving the
/// replication credentials (see [`datatable_shared_replication_resource`]).
///
/// Authorization: like its `_unchecked` sibling, returns resolved connection
/// credentials and performs no authorization — callers MUST have already authorized
/// access to the datatable (e.g. the trigger's own create-time check).
pub async fn get_datatable_replication_resource_from_db_unchecked(
db: &DB,
w_id: &str,
name: &str,
) -> Result<serde_json::Value> {
let datatable = get_datatable_config(db, w_id, name).await?;
datatable_shared_replication_resource(db, w_id, &datatable).await
}
#[derive(Deserialize, Serialize, Debug)]
pub struct Ducklake {
pub catalog: DucklakeCatalog,
backend/windmill-mcp/src/client/mod.rs
+13 -3
View File
@@ -43,7 +43,7 @@ impl McpClient {
// The resource URL is author-controlled and we send a (potentially
// secret) bearer token to it, so it must be validated against SSRF
// before we connect (e.g. cloud metadata endpoints, internal services).
windmill_common::ssrf::validate_mcp_server_url(&resource.url)
let validated = windmill_common::ssrf::validate_mcp_server_url(&resource.url)
.await
.map_err(|e| {
anyhow::anyhow!(
@@ -76,14 +76,24 @@ impl McpClient {
}
}
let reqwest_client = reqwest::Client::builder()
let mut client_builder = reqwest::Client::builder()
.default_headers(headers)
// Don't follow redirects: the SSRF check above only validates the
// initial (author-controlled) URL, so following a redirect could
// still reach a private/internal address with the bearer token
// attached. The MCP streamable-HTTP endpoint is a direct endpoint
// and does not legitimately rely on redirects.
.redirect(reqwest::redirect::Policy::none())
.redirect(reqwest::redirect::Policy::none());
// Pin DNS to the address validated above so the connect cannot rebind to
// an internal IP between the check and the request. `apply_dns_pinning`
// lives on windmill-common's reqwest, but this crate resolves a
// different reqwest version (via rmcp), so pin directly with the
// std-typed host/addrs the validation surfaced. Empty addrs (IP literal
// or ALLOW_PRIVATE_MCP_SERVER_URLS) leave resolution untouched.
if !validated.addrs.is_empty() {
client_builder = client_builder.resolve_to_addrs(&validated.host, &validated.addrs);
}
let reqwest_client = client_builder
.build()
.context("Failed to build HTTP client")?;
backend/windmill-mcp/src/client_registration.rs
+13 -5
View File
@@ -17,7 +17,7 @@ use windmill_common::db::DB;
use windmill_common::error;
use windmill_common::variables::{build_crypt, decrypt, encrypt};
use crate::oauth::{no_redirect_http_client, AuthorizationManager};
use crate::oauth::{no_redirect_http_client_pinned, AuthorizationManager};
/// MCP client credentials returned by [`get_or_refresh_mcp_client`].
pub struct McpClientCredentials {
@@ -77,13 +77,15 @@ async fn register_client(
redirect_uri: &str,
client_name: &str,
) -> Result<DcrResponse, error::Error> {
windmill_common::ssrf::validate_mcp_server_url_for_bad_request(
let validated = windmill_common::ssrf::validate_mcp_server_url_for_bad_request(
registration_endpoint,
"MCP server registration endpoint URL",
)
.await?;
let client = no_redirect_http_client()
// Pin to the validated address: DCR posts to an author-controlled endpoint,
// so the connect must not rebind to an internal IP after the check.
let client = no_redirect_http_client_pinned(&validated)
.map_err(|e| error::Error::BadRequest(format!("Failed to build DCR client: {e}")))?;
let request = DcrRequest {
client_name: client_name.to_string(),
@@ -128,7 +130,7 @@ pub async fn get_or_refresh_mcp_client(
let base_url = (**windmill_common::BASE_URL.load()).clone();
let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url);
windmill_common::ssrf::validate_mcp_server_url_for_bad_request(
let validated_server = windmill_common::ssrf::validate_mcp_server_url_for_bad_request(
mcp_server_url,
"MCP server URL",
)
@@ -166,7 +168,13 @@ pub async fn get_or_refresh_mcp_client(
let mut manager = AuthorizationManager::new(mcp_server_url)
.await
.map_err(|e| error::Error::BadRequest(format!("Failed to create auth manager: {e}")))?;
let discovery_client = no_redirect_http_client().map_err(|e| {
// Discovery hits the well-known endpoint on the MCP server host validated
// above; pin to that address so it cannot rebind between check and connect.
// Limitation: rmcp's discover_metadata may additionally follow server-supplied
// metadata URLs (resource_metadata / authorization_servers) on other hosts,
// which this per-host pin does not cover — a pre-existing gap in rmcp discovery
// that a validating resolver would need to close, out of scope for this pin.
let discovery_client = no_redirect_http_client_pinned(&validated_server).map_err(|e| {
error::Error::BadRequest(format!("Failed to build MCP OAuth discovery client: {e}"))
})?;
manager
backend/windmill-mcp/src/lib.rs
+21
View File
@@ -57,6 +57,27 @@ pub mod oauth {
.build()
}
/// Like [`no_redirect_http_client`], but pins DNS to the address the SSRF
/// guard validated for the request URL so the connect cannot rebind to an
/// internal IP after the check (TOCTOU). The OAuth DCR/discovery/token
/// requests target author-controlled URLs and carry secrets, so they must
/// go through this rather than the unpinned client. `apply_dns_pinning`
/// lives on windmill-common's reqwest, which this crate resolves at a
/// different version (via rmcp), so pin directly with the std-typed
/// host/addrs. Empty `addrs` (IP literal or ALLOW_PRIVATE_MCP_SERVER_URLS)
/// leaves resolution untouched.
pub fn no_redirect_http_client_pinned(
target: &windmill_common::ssrf::ValidatedTarget,
) -> Result<reqwest::Client, reqwest::Error> {
let mut builder = reqwest::Client::builder()
.timeout(DEFAULT_OAUTH_HTTP_TIMEOUT)
.redirect(reqwest::redirect::Policy::none());
if !target.addrs.is_empty() {
builder = builder.resolve_to_addrs(&target.host, &target.addrs);
}
builder.build()
}
// Re-export oauth2 types needed for MCP OAuth flow
pub use oauth2::{
basic::BasicClient, AuthUrl, ClientId, ClientSecret, CsrfToken, PkceCodeChallenge,
backend/windmill-object-store/src/lib.rs
+126 -10
View File
@@ -361,12 +361,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result<Arc<
|| s3_resource_ref.secret_key.as_ref().is_some_and(|x| x != "");
let credentials_provider = if !static_creds {
Some(
DefaultCredentialsChain::builder()
.region(Region::new(s3_resource_ref.region.clone()))
.build()
.await,
)
Some(ambient_aws_credentials_provider(&s3_resource_ref.region).await)
} else {
None
};
@@ -764,10 +759,92 @@ pub async fn build_s3_client_from_settings(
build_s3_client(&s3_resource).await
}
// Resolving the default chain goes over the network (ECS/IMDS) on instances relying on an
// instance role, and object_store asks its CredentialProvider on every request — so resolved
// credentials must be cached and only re-fetched when close to expiring.
#[cfg(feature = "parquet")]
#[derive(Debug)]
struct AmbientAwsCredentials {
chain: DefaultCredentialsChain,
cached: RwLock<Option<(aws_sdk_sts::config::Credentials, std::time::Instant)>>,
}
#[cfg(feature = "parquet")]
impl AmbientAwsCredentials {
// Credentials without an expiry (env vars, static profile) are still re-resolved
// periodically so runtime changes to the environment are eventually picked up.
const NO_EXPIRY_TTL: std::time::Duration = std::time::Duration::from_secs(300);
const EXPIRY_MARGIN: std::time::Duration = std::time::Duration::from_secs(120);
fn still_valid(creds: &aws_sdk_sts::config::Credentials, age: std::time::Duration) -> bool {
match creds.expiry() {
Some(expiry) => std::time::SystemTime::now() + Self::EXPIRY_MARGIN < expiry,
None => age < Self::NO_EXPIRY_TTL,
}
}
async fn get(&self) -> anyhow::Result<aws_sdk_sts::config::Credentials> {
if let Some((creds, fetched_at)) = self.cached.read().await.as_ref() {
if Self::still_valid(creds, fetched_at.elapsed()) {
return Ok(creds.clone());
}
}
// The write lock is held across the chain resolution so concurrent requests don't all
// hit the metadata service at once.
let mut guard = self.cached.write().await;
if let Some((creds, fetched_at)) = guard.as_ref() {
if Self::still_valid(creds, fetched_at.elapsed()) {
return Ok(creds.clone());
}
}
let creds = self.chain.provide_credentials().await.map_err(|e| {
anyhow::anyhow!(
"no S3 access key/secret key is configured and no ambient AWS credentials could \
be loaded through the AWS SDK default chain (env vars, profile, ECS/EC2 instance \
role): {cause}. If an EC2/ECS instance role is expected to be used, the instance \
metadata service must be reachable from the process running Windmill — on EC2 the \
AWS Rust SDK only supports IMDSv2, so when Windmill runs in a Docker container \
the instance metadata hop limit (HttpPutResponseHopLimit) must be at least 2",
cause = format!("{:#}", anyhow::Error::new(e))
)
})?;
*guard = Some((creds.clone(), std::time::Instant::now()));
Ok(creds)
}
}
#[cfg(feature = "parquet")]
lazy_static::lazy_static! {
static ref AMBIENT_AWS_CREDS_PROVIDERS: Cache<String, Arc<AmbientAwsCredentials>> =
Cache::new(20);
}
#[cfg(feature = "parquet")]
async fn ambient_aws_credentials_provider(region: &str) -> Arc<AmbientAwsCredentials> {
// Single-flight: concurrent cold misses for the same region must share one provider,
// otherwise each gets its own instance and their per-instance refresh locks can't serialize
// the initial credential resolution — every caller would hit the metadata service.
match AMBIENT_AWS_CREDS_PROVIDERS
.get_value_or_guard_async(region)
.await
{
Ok(provider) => provider,
Err(guard) => {
let chain = DefaultCredentialsChain::builder()
.region(Region::new(region.to_string()))
.build()
.await;
let provider = Arc::new(AmbientAwsCredentials { chain, cached: RwLock::new(None) });
let _ = guard.insert(provider.clone());
provider
}
}
}
#[cfg(feature = "parquet")]
#[derive(Debug)]
struct AwsCredentialAdapter {
pub inner: DefaultCredentialsChain,
pub inner: Arc<AmbientAwsCredentials>,
}
#[cfg(feature = "parquet")]
@@ -775,9 +852,9 @@ struct AwsCredentialAdapter {
impl CredentialProvider for AwsCredentialAdapter {
type Credential = AwsCredential;
async fn get_credential(&self) -> object_store::Result<Arc<Self::Credential>> {
let creds = self.inner.provide_credentials().await.map_err(|e| {
tracing::error!("Error getting credentials: {:?}", e);
object_store::Error::Generic { store: "AWS", source: Box::new(e) }
let creds = self.inner.get().await.map_err(|e| {
tracing::error!("Error getting AWS credentials: {e:#}");
object_store::Error::Generic { store: "AWS", source: e.into() }
})?;
Ok(Arc::new(Self::Credential {
key_id: creds.access_key_id().to_string(),
@@ -1481,6 +1558,45 @@ pub async fn get_logs_from_store(
mod tests {
use super::*;
// --- ambient credentials cache tests ---
#[cfg(feature = "parquet")]
#[test]
fn test_ambient_credentials_still_valid() {
use std::time::{Duration, SystemTime};
fn creds(expiry: Option<SystemTime>) -> aws_sdk_sts::config::Credentials {
let mut builder = aws_sdk_sts::config::Credentials::builder()
.access_key_id("AK")
.secret_access_key("SK")
.provider_name("test");
if let Some(expiry) = expiry {
builder = builder.expiry(expiry);
}
builder.build()
}
// Expiry far in the future: valid regardless of fetch time
assert!(AmbientAwsCredentials::still_valid(
&creds(Some(SystemTime::now() + Duration::from_secs(3600))),
Duration::ZERO
));
// Expiry within the refresh margin: must be re-fetched
assert!(!AmbientAwsCredentials::still_valid(
&creds(Some(SystemTime::now() + Duration::from_secs(30))),
Duration::ZERO
));
// No expiry: valid while fresh, re-fetched after the TTL
assert!(AmbientAwsCredentials::still_valid(
&creds(None),
Duration::ZERO
));
assert!(!AmbientAwsCredentials::still_valid(
&creds(None),
AmbientAwsCredentials::NO_EXPIRY_TTL + Duration::from_secs(1)
));
}
// --- render_endpoint tests ---
#[test]
backend/windmill-queue/src/jobs.rs
+323 -80
View File
@@ -25,6 +25,7 @@ use serde::{ser::SerializeMap, Serialize};
use serde_json::{json, value::RawValue};
use sqlx::{types::Json, Acquire, Pool, Postgres, Transaction};
use sqlx::{Encode, PgExecutor};
use std::borrow::Cow;
use tokio::sync::mpsc::Sender;
use tokio::sync::oneshot;
use tokio::task::JoinHandle;
@@ -49,7 +50,7 @@ use windmill_common::runnable_settings::{
RunnableSettings, RunnableSettingsTrait,
};
use windmill_common::triggers::TriggerMetadata;
use windmill_common::utils::{calculate_hash, configure_client, now_from_db};
use windmill_common::utils::{calculate_hash, configure_client, now_from_db, strip_json_nul};
use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY};
use windmill_common::otel_oss::{
@@ -634,6 +635,12 @@ pub trait ValidableJson {
fn wm_failure(&self) -> Option<String>;
fn result_metadata(&self) -> ResultMetadata;
fn size(&self) -> usize;
/// The result as JSON text, for binding into the `jsonb` `result` column.
/// `Box<RawValue>` is already serialized and returns a zero-cost borrow;
/// other impls serialize on demand. Callers pass this through
/// `strip_json_nul` before the INSERT, since a genuine NUL escape would
/// abort the write with 22P05.
fn serialized_json(&self) -> Cow<'_, str>;
}
/// The Windmill-specific markers we look for inside a job's result.
@@ -692,6 +699,10 @@ impl ValidableJson for WrappedError {
fn size(&self) -> usize {
0
}
fn serialized_json(&self) -> Cow<'_, str> {
Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string()))
}
}
impl ValidableJson for Box<RawValue> {
@@ -714,6 +725,11 @@ impl ValidableJson for Box<RawValue> {
fn size(&self) -> usize {
self.get().len()
}
fn serialized_json(&self) -> Cow<'_, str> {
// Already serialized JSON text — borrow it, no re-serialization.
Cow::Borrowed(self.get())
}
}
impl<T: ValidableJson> ValidableJson for Arc<T> {
@@ -736,6 +752,10 @@ impl<T: ValidableJson> ValidableJson for Arc<T> {
fn size(&self) -> usize {
T::size(&self)
}
fn serialized_json(&self) -> Cow<'_, str> {
T::serialized_json(&self)
}
}
impl ValidableJson for serde_json::Value {
@@ -758,6 +778,10 @@ impl ValidableJson for serde_json::Value {
fn size(&self) -> usize {
self.size_hint()
}
fn serialized_json(&self) -> Cow<'_, str> {
Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string()))
}
}
impl<T: ValidableJson> ValidableJson for Json<T> {
@@ -780,6 +804,10 @@ impl<T: ValidableJson> ValidableJson for Json<T> {
fn size(&self) -> usize {
self.0.size()
}
fn serialized_json(&self) -> Cow<'_, str> {
self.0.serialized_json()
}
}
pub async fn register_metric<T, F, F2, R>(
@@ -925,8 +953,13 @@ lazy_static::lazy_static! {
static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), (bool, Option<i32>)> = Cache::new(10000);
// Cache for workspace error handler settings with 60s TTL
// Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, expiry_timestamp)
static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, bool, bool, i64)> = Cache::new(1000);
// Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, report_to_instance_alerts, expiry_timestamp)
static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, bool, bool, bool, i64)> = Cache::new(1000);
// Best-effort per-worker throttle for the instance-channel fallback: a flapping runnable
// would otherwise turn every failure into outbound Slack/SMTP traffic on channels shared by
// the whole instance. Key: workspace_id, Value: (last_sent_epoch, failures suppressed since)
static ref INSTANCE_ALERT_THROTTLE: Cache<String, (i64, u64)> = Cache::new(1000);
// Cache for workspace success handler settings with 60s TTL
// Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp)
@@ -934,6 +967,7 @@ lazy_static::lazy_static! {
}
const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60;
const INSTANCE_ALERT_COOLDOWN_SECONDS: i64 = 60;
pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
db: &Pool<Postgres>,
@@ -1079,15 +1113,24 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// Resolve the concurrency-limit settings on the pool *before* opening the
// completion transaction: doing it inside the tx would hold a second
// simultaneous connection from the small per-worker pool.
let has_concurrent_limit = completed_job.concurrent_limit.is_some()
|| windmill_common::runnable_settings::prefetch_cached_from_handle(
completed_job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit
.is_some();
let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit)
|| has_active_concurrency_limit(
windmill_common::runnable_settings::prefetch_cached_from_handle(
completed_job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit,
);
// A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that
// the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape
// sequence"), which would abort the whole completion INSERT. Strip it before
// binding — near-zero cost when clean: a single scan, and for an
// already-serialized `RawValue` result the serialization itself is a borrow.
let serialized_result = result.serialized_json();
let sanitized_result = strip_json_nul(serialized_result.as_ref());
let mut tx = db.begin().warn_after_seconds(10).await?;
@@ -1107,7 +1150,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
, status
, worker
)
SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,
SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3::text::jsonb, $10, $5, $6,
flow_status, workflow_as_code_status,
$8, CASE WHEN $4::BOOL THEN 'canceled'::job_status
WHEN $7::BOOL THEN 'skipped'::job_status
@@ -1115,10 +1158,10 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
ELSE 'failure'::job_status END AS status,
q.worker
FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1
ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"",
ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb RETURNING duration_ms AS \"duration_ms!\"",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result as Json<&T>,
/* $3 */ sanitized_result.as_ref(),
/* $4 */ canceled_by.is_some(),
/* $5 */ canceled_by.clone().map(|cb| cb.username).flatten(),
/* $6 */ canceled_by.clone().map(|cb| cb.reason).flatten(),
@@ -1156,7 +1199,14 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
}
};
if let Some(labels) = result.wm_labels() {
if let Some(mut labels) = result.wm_labels() {
// A `\u0000` inside a wm_labels entry decodes to a real NUL that the
// `text[]` column rejects, which would abort this same transaction (and
// roll back the sanitized result insert) exactly like an unsanitized
// result. Strip it so the labels match the sanitized result.
for label in &mut labels {
label.retain(|c| c != '\0');
}
sqlx::query!(
"UPDATE v2_job SET labels = (
SELECT array_agg(DISTINCT all_labels)
@@ -2081,7 +2131,7 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel(
async fn fetch_error_handler_from_db(
db: &Pool<Postgres>,
w_id: &str,
) -> Result<(Option<String>, Option<Json<Box<RawValue>>>, bool, bool), Error> {
) -> Result<(Option<String>, Option<Json<Box<RawValue>>>, bool, bool, bool), Error> {
sqlx::query_as::<
_,
(
@@ -2089,6 +2139,7 @@ async fn fetch_error_handler_from_db(
Option<Json<Box<RawValue>>>,
Option<bool>,
Option<bool>,
bool,
),
>(
r#"
@@ -2096,23 +2147,28 @@ async fn fetch_error_handler_from_db(
error_handler->>'path',
(error_handler->'extra_args')::text::json,
(error_handler->>'muted_on_cancel')::boolean,
(error_handler->>'muted_on_user_path')::boolean
FROM workspace_settings
WHERE workspace_id = $1
(error_handler->>'muted_on_user_path')::boolean,
ws.error_handler_fallback_to_instance_alerts AND w.parent_workspace_id IS NULL
FROM workspace_settings ws
JOIN workspace w ON w.id = ws.workspace_id
WHERE ws.workspace_id = $1
"#,
)
.bind(w_id)
.fetch_optional(db)
.await
.context("fetching error handler info from workspace_settings")?
.map(|(path, extra_args, muted_on_cancel, muted_on_user_path)| {
(
path,
extra_args,
muted_on_cancel.unwrap_or(false),
muted_on_user_path.unwrap_or(false),
)
})
.map(
|(path, extra_args, muted_on_cancel, muted_on_user_path, report_to_instance_alerts)| {
(
path,
extra_args,
muted_on_cancel.unwrap_or(false),
muted_on_user_path.unwrap_or(false),
report_to_instance_alerts,
)
},
)
.ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))
}
@@ -2130,15 +2186,22 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
error_handler_extra_args,
error_handler_muted_on_cancel,
error_handler_muted_on_user_path,
report_to_instance_alerts,
) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) {
if cached.4 > now {
(cached.0.clone(), cached.1.clone(), cached.2, cached.3)
if cached.5 > now {
(
cached.0.clone(),
cached.1.clone(),
cached.2,
cached.3,
cached.4,
)
} else {
let row = fetch_error_handler_from_db(db, w_id).await?;
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_ERROR_HANDLER_CACHE.insert(
w_id.clone(),
(row.0.clone(), row.1.clone(), row.2, row.3, expiry),
(row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry),
);
row
}
@@ -2147,11 +2210,17 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
WORKSPACE_ERROR_HANDLER_CACHE.insert(
w_id.clone(),
(row.0.clone(), row.1.clone(), row.2, row.3, expiry),
(row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry),
);
row
};
// Nothing to do for the vast majority of workspaces, and returning here keeps the
// per-runnable mute lookup below off the path of every failed job.
if error_handler.is_none() && !report_to_instance_alerts {
return Ok(());
}
if is_canceled && error_handler_muted_on_cancel {
return Ok(());
}
@@ -2165,51 +2234,90 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
}
}
if let Some(error_handler) = error_handler {
let ws_error_handler_muted: Option<bool> = match queued_job.kind {
JobKind::Script => {
sqlx::query_scalar!(
let ws_error_handler_muted: Option<bool> = match queued_job.kind {
JobKind::Script => {
sqlx::query_scalar!(
"SELECT ws_error_handler_muted FROM script WHERE workspace_id = $1 AND hash = $2",
queued_job.workspace_id,
queued_job.runnable_id.map(|x| x.0),
)
.fetch_optional(db)
.await?
}
JobKind::Flow => {
sqlx::query_scalar!(
"SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2",
queued_job.workspace_id,
queued_job.runnable_path.clone(),
)
.fetch_optional(db)
.await?
}
_ => None,
};
let muted = ws_error_handler_muted.unwrap_or(false);
if !muted {
tracing::info!("workspace error handled for job {}", &queued_job.id);
push_error_handler(
db,
queued_job.id,
queued_job.schedule_path(),
.fetch_optional(db)
.await?
}
JobKind::Flow => {
sqlx::query_scalar!(
"SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2",
queued_job.workspace_id,
queued_job.runnable_path.clone(),
queued_job.is_flow(),
&queued_job.workspace_id,
&error_handler,
result,
None,
queued_job.started_at,
error_handler_extra_args,
&queued_job.permissioned_as_email,
false,
false,
None,
)
.await?;
.fetch_optional(db)
.await?
}
_ => None,
};
if ws_error_handler_muted.unwrap_or(false) {
return Ok(());
}
if let Some(error_handler) = error_handler {
tracing::info!("workspace error handled for job {}", &queued_job.id);
push_error_handler(
db,
queued_job.id,
queued_job.schedule_path(),
queued_job.runnable_path.clone(),
queued_job.is_flow(),
&queued_job.workspace_id,
&error_handler,
result,
None,
queued_job.started_at,
error_handler_extra_args,
&queued_job.permissioned_as_email,
false,
false,
None,
)
.await?;
} else if !is_canceled {
// A cancellation is a human action rather than an operational failure, and unlike the
// handler path this one has no per-workspace toggle to opt out of reporting them.
let suppressed = match INSTANCE_ALERT_THROTTLE.get(w_id) {
Some((last_sent, suppressed))
if now - last_sent < INSTANCE_ALERT_COOLDOWN_SECONDS =>
{
INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (last_sent, suppressed + 1));
None
}
entry => {
INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (now, 0));
Some(entry.map(|(_, suppressed)| suppressed).unwrap_or(0))
}
};
if let Some(suppressed) = suppressed {
tracing::info!(
"reporting failed job {} to the instance critical alert channels",
&queued_job.id
);
let base_url = windmill_common::BASE_URL.load();
let rollup = if suppressed > 0 {
format!(
" (and {suppressed} more failure(s) in the preceding {INSTANCE_ALERT_COOLDOWN_SECONDS}s)"
)
} else {
String::new()
};
windmill_common::utils::send_workspace_error_to_instance_channels(
format!(
"Job {} failed in workspace {w_id} ({base_url}/run/{}?workspace={w_id}){rollup}",
queued_job.runnable_path.as_deref().unwrap_or("preview"),
queued_job.id
),
db,
)
.await;
}
}
Ok(())
@@ -3878,7 +3986,7 @@ pub async fn pull(
let pulled_job_result = match job {
#[cfg(feature = "private")]
Some(job)
if concurrency_settings.concurrent_limit.is_some()
if has_active_concurrency_limit(concurrency_settings.concurrent_limit)
// Concurrency limit is available for either enterprise job or dependency job
&& (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING)) =>
{
@@ -3942,7 +4050,8 @@ pub async fn pull(
.1
.maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s);
let has_concurent_limit = concurrency_settings.concurrent_limit.is_some();
let has_concurent_limit =
has_active_concurrency_limit(concurrency_settings.concurrent_limit);
#[cfg(not(feature = "enterprise"))]
if has_concurent_limit && !job.is_dependency() {
@@ -3951,7 +4060,7 @@ pub async fn pull(
#[cfg(not(feature = "enterprise"))]
let has_concurent_limit = job.is_dependency()
&& job.concurrent_limit.is_some()
&& has_active_concurrency_limit(job.concurrent_limit)
&& cfg!(feature = "private")
&& !*WMDEBUG_NO_DEBOUNCING;
// if we don't have private flag, we don't have concurrency limit
@@ -4119,6 +4228,13 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
Ok(job_and_suspended)
}
/// A concurrency limit is only active when it caps at 1+ slots. `Some(0)` (or negative) is
/// a disabled limit, not a zero-slot one — see [`ConcurrencySettings::normalized`]. The gate
/// checks must use this instead of `.is_some()` so a legacy stored `0` behaves as disabled.
pub fn has_active_concurrency_limit(concurrent_limit: Option<i32>) -> bool {
concurrent_limit.is_some_and(|n| n > 0)
}
pub async fn custom_concurrency_key(
db: &Pool<Postgres>,
job_id: &Uuid,
@@ -5614,6 +5730,10 @@ async fn push_inner<'c, 'd>(
restarted_from_val.step_id.as_str(),
restarted_from_val.branch_or_iteration_n,
restarted_from_val.flow_version,
restarted_from_val.nested.is_some(),
// RawFlow queues the request's (possibly edited) definition, not the
// stored one, so zombie reuse of the stored step is unsafe here.
false,
)
.await?;
FlowStatus {
@@ -6003,6 +6123,10 @@ async fn push_inner<'c, 'd>(
step_id.as_str(),
branch_or_iteration_n,
flow_version,
nested.is_some(),
// RestartedFlow resolves and queues the completed job's stored definition, so the
// step validated for reuse is the one that will run.
true,
)
.await?;
@@ -6098,6 +6222,11 @@ async fn push_inner<'c, 'd>(
},
};
// Guard against an already-stored `concurrent_limit <= 0` reaching the queue: it would
// register a zero-slot concurrency key and permanently block the job. Coerce it to
// disabled before it is persisted onto the job row / concurrency key here.
concurrency_settings = concurrency_settings.normalized();
// Enforce concurrency limit on all dependency jobs.
// TODO: We can ignore this for scripts djobs. The main reason we need all djobs to be sequential is because we have
// nodes_to_relock and we need all locks whose corresponding steps aren't in nodes_to_relock be already present.
@@ -6285,7 +6414,7 @@ async fn push_inner<'c, 'd>(
check_workspace_queue_cap(&mut *tx, workspace_id).await?;
}
if concurrency_settings.concurrent_limit.is_some() {
if has_active_concurrency_limit(concurrency_settings.concurrent_limit) {
let concurrency_key = resolve_concurrency_key(
workspace_id,
&args,
@@ -6689,7 +6818,7 @@ pub async fn insert_concurrency_key_capped<'d, 'c, E: PgExecutor<'c> + Copy>(
custom_concurrency_key,
);
#[cfg(feature = "cloud")]
if *CLOUD_HOSTED && concurrent_limit.is_some() {
if *CLOUD_HOSTED && has_active_concurrency_limit(concurrent_limit) {
check_concurrency_key_queue_cap(db, &concurrency_key).await?;
}
#[cfg(not(feature = "cloud"))]
@@ -7006,6 +7135,78 @@ fn create_restarted_module(
}
}
/// A between-steps-zombie step: an `InProgress` module (in an otherwise terminal,
/// reaped flow) whose every child is recorded as a `success` completion. Only the
/// module's final state transition was lost, so the whole step is derivable and
/// safe to reuse on restart. Children incomplete/failed/cancelled ⟹ not a zombie.
async fn is_derivable_between_steps_zombie(
db: &Pool<Postgres>,
workspace_id: &str,
module: &FlowStatusModule,
) -> Result<bool, Error> {
// The module's own cursor must prove it reached the end (a serial loop/branch-all reaped
// mid-fan-out has an all-success prefix but unrun remaining iterations); while-loops are
// never derivable. Children-success is verified below.
if !module.is_between_steps_complete() {
return Ok(false);
}
let child_ids: Vec<Uuid> = module
.flow_jobs()
.filter(|v| !v.is_empty())
.or_else(|| module.job().map(|j| vec![j]))
.unwrap_or_default();
if child_ids.is_empty() {
return Ok(false);
}
let success_children = sqlx::query_scalar!(
"SELECT count(*) FROM v2_job_completed
WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'",
workspace_id,
&child_ids,
)
.fetch_one(db)
.await?
.unwrap_or(0);
Ok(success_children == child_ids.len() as i64)
}
/// Convert a between-steps-zombie `InProgress` module (validated by
/// [`is_derivable_between_steps_zombie`]) into the `Success` it would have become
/// had its dropped transition landed, reusing all completed children. Downstream
/// steps re-derive this step's result from `flow_jobs`/`job` on demand
/// (`get_previous_job_result`), so no aggregate needs recomputing here.
fn reuse_completed_zombie_module(module: FlowStatusModule) -> FlowStatusModule {
match module {
FlowStatusModule::InProgress {
id,
job,
flow_jobs,
flow_jobs_success,
flow_jobs_duration,
branch_chosen,
agent_actions,
agent_actions_success,
..
} => FlowStatusModule::Success {
id,
job,
// Every child was verified successful, so normalise the success
// vector (the dropped transition may have left the last entry unset).
flow_jobs_success: flow_jobs_success
.map(|v| v.into_iter().map(|_| Some(true)).collect()),
flow_jobs,
flow_jobs_duration,
branch_chosen,
approvers: vec![],
failed_retries: vec![],
skipped: false,
agent_actions,
agent_actions_success,
},
other => other,
}
}
async fn restarted_flows_resolution(
db: &Pool<Postgres>,
workspace_id: &str,
@@ -7013,6 +7214,15 @@ async fn restarted_flows_resolution(
restart_step_id: &str,
branch_or_iteration_n: Option<usize>,
flow_version: Option<i64>,
// A nested restart chain (RestartedFrom.nested) descends into the restart step's child to
// re-run an inner step; zombie reuse would skip the whole container and ignore it.
nested_restart: bool,
// Zombie reuse validates the restart step against the completed job's STORED definition and
// synthesizes Success from its recorded children. That is only sound when the run being queued
// uses that same definition (JobPayload::RestartedFlow). A JobPayload::RawFlow restart queues
// the editor's current, possibly EDITED, definition instead, so reuse would skip the edited
// step and reuse the old child result; disable it there.
allow_zombie_reuse: bool,
) -> Result<
(
Option<i64>,
@@ -7029,7 +7239,7 @@ async fn restarted_flows_resolution(
let row = sqlx::query!(
"SELECT
j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",
j.kind AS \"job_kind!: JobKind\",
j.kind AS \"job_kind!: JobKind\", c.canceled_by,
COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json<Box<RawValue>>\",
j.raw_flow AS \"raw_flow: Json<Box<RawValue>>\"
FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2",
@@ -7045,6 +7255,12 @@ async fn restarted_flows_resolution(
))
})?;
// Zombie reuse must only apply to flows the zombie monitor reaped (canceled_by = 'monitor').
// An ordinary force-cancel copies the same live flow_status, so a user canceling after a child
// succeeds but before the parent transition lands produces the identical InProgress/all-success
// shape; those must retain restart-from-step semantics (the step re-runs).
let reaped_by_monitor = row.canceled_by.as_deref() == Some("monitor");
let current_flow_version = row.script_hash.map(|x| x.0);
let is_version_change = flow_version.is_some()
&& current_flow_version.is_some()
@@ -7135,9 +7351,36 @@ async fn restarted_flows_resolution(
continue;
};
if module.id() == restart_step_id {
// if the module ID is the one we want to restart the flow at, or if it's past it in the flow,
// set the module as WaitingForPriorSteps as it needs to be re-run
if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 {
// Reuse is only safe when there is a NEXT step to advance into (advancing past the
// last module lands on the failure step) and the step's definition carries no
// completion/arming semantics that reuse would skip (stop predicates, skip_if,
// suspend, sleep); such a step must re-run, not be synthesized as Success.
let has_next_step = flow_value
.modules
.last()
.is_none_or(|m| m.id != restart_step_id);
// A whole-step restart is `None` (restart API with the field omitted) or `Some(0)`
// (the run page's "Re-start from" button always sends 0); both mean "redo this step",
// which for a monitor-reaped zombie means reuse it. `Some(n>=1)` is an explicit
// partial container restart and keeps its existing reuse-0..n-1 / rerun-from-n path.
if allow_zombie_reuse
&& reaped_by_monitor
&& branch_or_iteration_n.unwrap_or(0) == 0
&& !nested_restart
&& has_next_step
&& module_definition.allows_zombie_reuse()
&& is_derivable_between_steps_zombie(db, workspace_id, &module).await?
{
// Between-steps-zombie recovery: this step's children all
// completed but its final state transition was dropped (the
// flow was reaped by the zombie monitor). Reuse the completed
// step verbatim and restart from the NEXT step, so no child
// re-runs and only the dropped transition is replayed onward.
step_n += 1;
truncated_modules.push(reuse_completed_zombie_module(module));
} else if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 {
// if the module ID is the one we want to restart the flow at, or if it's past it in the flow,
// set the module as WaitingForPriorSteps as it needs to be re-run
// The module as WaitingForPriorSteps as the entire module (i.e. all the branches) need to be re-run
truncated_modules
.push(FlowStatusModule::WaitingForPriorSteps { id: module.id() });
backend/windmill-queue/tests/concurrency_limit_zero_test.rs
+21
View File
@@ -0,0 +1,21 @@
//! Runtime gate for the `Some(0)` concurrency footgun: a stored `concurrent_limit <= 0`
//! must read as "disabled", never as a zero-slot cap that permanently blocks the job at the
//! concurrency gate (the re-queue storm the zombie monitor eventually fails as a fake OOM).
//!
//! Run with:
//! cargo test -p windmill-queue --test concurrency_limit_zero_test
use windmill_queue::jobs::has_active_concurrency_limit;
#[test]
fn zero_and_negative_are_not_active_limits() {
assert!(!has_active_concurrency_limit(None));
assert!(!has_active_concurrency_limit(Some(0)));
assert!(!has_active_concurrency_limit(Some(-1)));
}
#[test]
fn positive_limit_is_active() {
assert!(has_active_concurrency_limit(Some(1)));
assert!(has_active_concurrency_limit(Some(i32::MAX)));
}
backend/windmill-store/src/resources.rs
+4 -4
View File
@@ -184,6 +184,7 @@ struct EditResource {
path: Option<String>,
description: Option<String>,
value: Option<Box<RawValue>>,
resource_type: Option<String>,
labels: Option<Vec<String>>,
ws_specific: Option<bool>,
}
@@ -242,12 +243,8 @@ async fn list_search_resources(
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<SearchResource>> {
let mut tx = user_db.begin(&authed).await?;
#[cfg(feature = "enterprise")]
let n = 1000;
#[cfg(not(feature = "enterprise"))]
let n = 3;
let allowed = build_scope_path_predicate(&authed, "resources", "read");
let rows = sqlx::query_as!(
SearchResource,
@@ -1717,6 +1714,9 @@ async fn update_resource(
if let Some(nvalue) = &ns.value {
sqlb.set_str("value", nvalue.to_string());
}
if let Some(nrt) = &ns.resource_type {
sqlb.set_str("resource_type", nrt);
}
if let Some(ndesc) = ns.description {
sqlb.set_str("description", ndesc);
}
backend/windmill-trigger-postgres/src/lib.rs
+7 -5
View File
@@ -382,10 +382,9 @@ pub async fn resolve_postgres_resource(
) -> Result<Postgres> {
if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") {
// Every trigger-side surface (capture, publication/slot management,
// the CDC listener itself) connects through the shared owner role and
// reads or manages all change data — so on a permissions-enabled data
// table this whole resolution is admin-only, mirroring the
// create/edit gate.
// the CDC listener itself) reads or manages all change data through a
// shared role — so on a permissions-enabled data table this whole
// resolution is admin-only, mirroring the create/edit gate.
let config =
windmill_common::workspaces::get_datatable_config(db, w_id, datatable_name).await?;
if windmill_common::datatable_permissions::datatable_permissions_enabled(&config)
@@ -397,8 +396,11 @@ pub async fn resolve_postgres_resource(
read all database changes through the shared role."
)));
}
// Trigger connections (publication/slot management + logical replication) run
// as the dedicated replication user on custom-instance databases.
let resource_value =
windmill_common::workspaces::datatable_shared_resource(db, w_id, &config).await?;
windmill_common::workspaces::datatable_shared_replication_resource(db, w_id, &config)
.await?;
serde_json::from_value::<Postgres>(resource_value).map_err(|e| Error::SerdeJson {
error: e,
location: "resolve_postgres_resource".to_string(),
backend/windmill-trigger-websocket/src/handler.rs
+2 -2
View File
@@ -316,9 +316,9 @@ impl TriggerCrud for WebsocketTrigger {
Cow::Borrowed(&url)
};
validate_websocket_url_for_ssrf(&connect_url).await?;
let validated = validate_websocket_url_for_ssrf(&connect_url).await?;
connect_async_with_proxy(&*connect_url)
connect_async_with_proxy(&*connect_url, validated.pinned_addrs())
.await
.map_err(|err| {
Error::BadConfig(format!(
backend/windmill-trigger-websocket/src/lib.rs
+23 -8
View File
@@ -119,14 +119,15 @@ pub const ALLOW_PRIVATE_WEBSOCKET_URLS_ENV: &str = "ALLOW_PRIVATE_WEBSOCKET_URLS
/// `$flow:`/`$script:` URL is checked on its returned value and re-checked on
/// each reconnect (DNS rebinding). `validate_config` also calls this at save
/// time to reject static URLs early.
pub async fn validate_websocket_url_for_ssrf(url: &str) -> Result<()> {
if std::env::var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV)
.ok()
.is_some_and(|v| v == "true" || v == "1")
{
return Ok(());
}
///
/// Returns the resolved [`ValidatedTarget`]: the connect must pin these
/// addresses (see [`proxy::connect_async_with_proxy`]) so a rebinder cannot swap
/// in an internal IP between this check and the connect (TOCTOU). The returned
/// `addrs` carry the http(s) port, which equals the ws(s) port the connection
/// uses (ws→80, wss→443, or the explicit port preserved through the mapping).
pub async fn validate_websocket_url_for_ssrf(
url: &str,
) -> Result<windmill_common::ssrf::ValidatedTarget> {
// `ws`/`wss` aren't recognised by `validate_url_for_ssrf`'s scheme check, so
// map them to the http(s) equivalent the same connection would tunnel over.
// The prefixes are ASCII, so byte-slicing at their length stays on a char
@@ -140,6 +141,20 @@ pub async fn validate_websocket_url_for_ssrf(url: &str) -> Result<()> {
url.to_string()
};
if std::env::var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV)
.ok()
.is_some_and(|v| v == "true" || v == "1")
{
// Opted out of the SSRF guard: allow any host and pin nothing. Parse the
// host for a uniform return; fall back to an empty target if unparseable
// (the connect then resolves normally, matching the pre-guard behavior).
let host = url::Url::parse(&http_url)
.ok()
.and_then(|u| u.host_str().map(str::to_string))
.unwrap_or_default();
return Ok(windmill_common::ssrf::ValidatedTarget { host, addrs: Vec::new() });
}
windmill_common::ssrf::validate_url_for_ssrf(&http_url)
.await
.map_err(|e| match e {
backend/windmill-trigger-websocket/src/listener.rs
+2 -2
View File
@@ -191,7 +191,7 @@ impl Listener for WebsocketTrigger {
Cow::Borrowed(&url)
};
validate_websocket_url_for_ssrf(&connect_url).await?;
let validated = validate_websocket_url_for_ssrf(&connect_url).await?;
// Gateway endpoints are often fronted by an edge proxy (e.g. Cloudflare)
// that sporadically answers the upgrade request with a transient 5xx
@@ -203,7 +203,7 @@ impl Listener for WebsocketTrigger {
let mut attempt = 0;
loop {
attempt += 1;
match connect_async_with_proxy(&*connect_url).await {
match connect_async_with_proxy(&*connect_url, validated.pinned_addrs()).await {
Ok(conn) => return Ok(Some(conn)),
// Only retry in trigger mode: a failed connect there disables the
// trigger until a human re-enables it, while capture mode is an
backend/windmill-trigger-websocket/src/proxy.rs
+57 -24
View File
@@ -15,6 +15,7 @@
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use std::io;
use std::net::SocketAddr;
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
net::TcpStream,
@@ -33,19 +34,28 @@ use windmill_common::{HTTPS_PROXY, HTTP_PROXY, NO_PROXY};
/// Drop-in replacement for `tokio_tungstenite::connect_async` that routes
/// the underlying TCP connection through `HTTPS_PROXY` / `HTTP_PROXY`
/// (with `NO_PROXY` exclusions) when those env vars are set. When none
/// is set we short-circuit straight to `connect_async`, keeping the
/// behaviour for non-proxied deployments unchanged.
/// (with `NO_PROXY` exclusions) when those env vars are set, and — for direct
/// (non-proxied) connections — pins DNS to `pinned_addrs`.
///
/// `pinned_addrs` are the addresses the SSRF guard already resolved and
/// validated for this URL (see `validate_websocket_url_for_ssrf`). Connecting
/// straight to them, rather than letting `connect_async` re-resolve the host,
/// closes the DNS-rebinding window between the check and the connect: a rebinder
/// cannot answer a public IP at validation time and an internal one here. When
/// `pinned_addrs` is empty (IP-literal host, or the SSRF guard opted out via
/// `ALLOW_PRIVATE_WEBSOCKET_URLS`) there is nothing to pin and we fall back to
/// `connect_async`, keeping the behaviour for those cases unchanged.
///
/// When a proxy applies, the proxy itself resolves the target host, so DNS
/// rebinding at this hop is not the worker's concern and `pinned_addrs` is
/// unused for that path.
pub async fn connect_async_with_proxy<R>(
request: R,
pinned_addrs: &[SocketAddr],
) -> Result<(WebSocketStream<MaybeTlsStream<TcpStream>>, Response), WsError>
where
R: IntoClientRequest + Unpin,
{
if HTTPS_PROXY.is_none() && HTTP_PROXY.is_none() {
return connect_async(request).await;
}
let request = request.into_client_request()?;
let uri = request.uri().clone();
let scheme = uri.scheme_str().unwrap_or_default().to_ascii_lowercase();
@@ -62,26 +72,49 @@ where
})
.ok_or(WsError::Url(UrlError::UnsupportedUrlScheme))?;
let proxy = proxy_url_for(&scheme, &host).and_then(|raw| parse_proxy_target(&raw));
let Some(proxy) = proxy else {
// Proxy env was set but doesn't apply to this host (NO_PROXY hit
// or unparseable URL): preserve the original connect path.
return connect_async(request).await;
let proxy = if HTTPS_PROXY.is_none() && HTTP_PROXY.is_none() {
None
} else {
proxy_url_for(&scheme, &host).and_then(|raw| parse_proxy_target(&raw))
};
tracing::debug!(
"Connecting to WebSocket {}:{} through HTTP proxy {}:{}",
host,
port,
proxy.host,
proxy.port,
);
let socket = http_connect_tunnel(&proxy, &host, port)
.await
.map_err(WsError::Io)?;
if let Some(proxy) = proxy {
tracing::debug!(
"Connecting to WebSocket {}:{} through HTTP proxy {}:{}",
host,
port,
proxy.host,
proxy.port,
);
let socket = http_connect_tunnel(&proxy, &host, port)
.await
.map_err(WsError::Io)?;
return client_async_tls_with_config(request, socket, None, None).await;
}
client_async_tls_with_config(request, socket, None, None).await
// Direct connection. Nothing to pin (IP literal or SSRF guard opted out):
// preserve the original resolve-and-connect path.
if pinned_addrs.is_empty() {
return connect_async(request).await;
}
// Pin to a validated address so this connect targets the same IP the SSRF
// guard checked. Try each in order (e.g. IPv6 then IPv4) until one connects.
let mut last_err: Option<io::Error> = None;
for addr in pinned_addrs {
match TcpStream::connect(addr).await {
Ok(socket) => {
return client_async_tls_with_config(request, socket, None, None).await;
}
Err(e) => last_err = Some(e),
}
}
Err(WsError::Io(last_err.unwrap_or_else(|| {
io::Error::new(
io::ErrorKind::AddrNotAvailable,
"no pinned address to connect",
)
})))
}
#[derive(Debug, Clone, PartialEq, Eq)]
backend/windmill-types/src/flow_status.rs
+154
View File
@@ -475,6 +475,33 @@ impl FlowStatusModule {
}
}
/// For a still-`InProgress` module (a between-steps zombie), whether the module's own
/// iteration/branch cursor proves it actually reached the end, so the only thing left is
/// the final state transition (children-success is a separate, DB-side check).
///
/// A serial for-loop / branch-all grows `flow_jobs` one entry at a time, so an all-success
/// prefix does NOT mean the module finished: the cursor must sit on the last element. Parallel
/// containers preallocate every child up front, so a full success set is conclusive. While-loops
/// are never derivable here (continuation depends on a condition evaluated after each iteration,
/// which a reaped zombie never persisted). Non-`InProgress` modules return false.
pub fn is_between_steps_complete(&self) -> bool {
match self {
FlowStatusModule::InProgress { while_loop: true, .. } => false,
// Parallel loop/branch-all: all children exist up front, so children-success suffices.
FlowStatusModule::InProgress { parallel: true, .. } => true,
FlowStatusModule::InProgress { iterator: Some(it), .. } => {
let total = it
.itered_len
.or_else(|| it.itered.as_ref().map(|v| v.len()));
total.is_some_and(|t| t > 0 && it.index + 1 == t)
}
FlowStatusModule::InProgress { branchall: Some(ba), .. } => ba.branch + 1 == ba.len,
// Single-child leaf / subflow / branch-one: the child ran, nothing else to advance.
FlowStatusModule::InProgress { .. } => true,
_ => false,
}
}
pub fn agent_actions(&self) -> Option<Vec<AgentAction>> {
match self {
FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(),
@@ -549,4 +576,131 @@ impl FlowStatus {
let i = usize::try_from(self.step).ok()?;
self.modules.get(i)
}
/// Whether no step has begun executing yet: the preprocessor (if any) and the first
/// module are both still `WaitingForPriorSteps`. A reaped flow in this state can be
/// safely re-queued because nothing ran. A preprocessor that is `InProgress` means its
/// child already ran (only the parent transition was lost), so re-queuing would
/// re-run the preprocessor and duplicate its side effects.
pub fn is_not_yet_started(&self) -> bool {
self.preprocessor_module
.as_ref()
.is_none_or(|p| matches!(p, FlowStatusModule::WaitingForPriorSteps { .. }))
&& self
.modules
.first()
.is_some_and(|m| matches!(m, FlowStatusModule::WaitingForPriorSteps { .. }))
}
}
#[cfg(test)]
mod tests {
use super::{FlowStatus, FlowStatusModule};
fn module(json: serde_json::Value) -> FlowStatusModule {
serde_json::from_value(json).unwrap()
}
fn status(json: serde_json::Value) -> FlowStatus {
serde_json::from_value(json).unwrap()
}
#[test]
fn is_not_yet_started_distinguishes_preprocessor_zombie() {
let nil = "00000000-0000-0000-0000-000000000000";
let waiting = serde_json::json!({ "type": "WaitingForPriorSteps", "id": "a" });
let failure = serde_json::json!({ "type": "WaitingForPriorSteps", "id": "failure" });
// No preprocessor, first module waiting: genuinely unstarted.
assert!(status(serde_json::json!({
"step": 0, "modules": [waiting], "failure_module": failure
}))
.is_not_yet_started());
// First module already InProgress: started.
assert!(!status(serde_json::json!({
"step": 0,
"modules": [{ "type": "InProgress", "id": "a", "job": nil }],
"failure_module": failure
}))
.is_not_yet_started());
// Preprocessor still waiting, first module waiting: unstarted.
assert!(status(serde_json::json!({
"step": -1, "modules": [waiting], "failure_module": failure,
"preprocessor_module": { "type": "WaitingForPriorSteps", "id": "pre" }
}))
.is_not_yet_started());
// Preprocessor InProgress (its child ran) while modules[0] still waits: a
// preprocessor zombie, NOT unstarted, so it must not be auto-requeued.
assert!(!status(serde_json::json!({
"step": -1, "modules": [waiting], "failure_module": failure,
"preprocessor_module": { "type": "InProgress", "id": "pre", "job": nil }
}))
.is_not_yet_started());
}
#[test]
fn between_steps_complete_serial_loop() {
// Cursor on the last iteration => complete.
assert!(module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"iterator": { "index": 1, "itered_len": 2 }, "flow_jobs": []
}))
.is_between_steps_complete());
// Reaped mid-iteration (iteration 1 of 2 never scheduled) => NOT complete.
assert!(!module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"iterator": { "index": 0, "itered_len": 2 }, "flow_jobs": []
}))
.is_between_steps_complete());
// Legacy shape: itered array present, itered_len absent.
assert!(module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"iterator": { "index": 1, "itered": ["x", "y"] }
}))
.is_between_steps_complete());
}
#[test]
fn between_steps_complete_while_loop_never() {
assert!(!module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"while_loop": true, "iterator": { "index": 1, "itered_len": 2 }
}))
.is_between_steps_complete());
}
#[test]
fn between_steps_complete_branchall_and_parallel() {
// Serial branch-all on the last branch => complete; earlier branch => not.
assert!(module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"branchall": { "branch": 1, "len": 2 }
}))
.is_between_steps_complete());
assert!(!module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"branchall": { "branch": 0, "len": 2 }
}))
.is_between_steps_complete());
// Parallel loop: children preallocated, so any cursor is fine (success is checked elsewhere).
assert!(module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"parallel": true, "iterator": { "index": 0, "itered_len": 3 }
}))
.is_between_steps_complete());
}
#[test]
fn between_steps_complete_leaf_and_non_inprogress() {
// Single-child leaf: the child ran, nothing to advance.
assert!(module(serde_json::json!({
"type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000"
}))
.is_between_steps_complete());
// A Success module is not a between-steps zombie.
assert!(!module(serde_json::json!({
"type": "Success", "id": "a", "job": "00000000-0000-0000-0000-000000000000",
"skipped": false
}))
.is_between_steps_complete());
}
}
backend/windmill-types/src/flows.rs
+13
View File
@@ -664,6 +664,19 @@ impl FlowModule {
.is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript")
}
/// Whether a between-steps-zombie step carrying this definition can be safely reused as
/// `Success` on restart (see restart-resolution reuse). Excludes steps whose completion
/// transition or arming carries semantics that reuse would silently skip: stop predicates
/// (`stop_after_if` / `stop_after_all_iters_if`, which decide whether downstream steps run),
/// `skip_if` (skipped-state and suspend arming), a `suspend` approval boundary, and `sleep`.
pub fn allows_zombie_reuse(&self) -> bool {
self.stop_after_if.is_none()
&& self.stop_after_all_iters_if.is_none()
&& self.skip_if.is_none()
&& self.suspend.is_none()
&& self.sleep.is_none()
}
pub fn get_type(&self) -> anyhow::Result<&str> {
#[derive(Deserialize)]
pub struct FlowModuleValueType<'a> {
backend/windmill-types/src/runnable_settings.rs
+186 -4
View File
@@ -93,9 +93,7 @@ pub struct DebouncingSettings {
pub debounce_args_to_accumulate: Option<Vec<String>>,
}
#[derive(
Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode,
)]
#[derive(Debug, Default, Clone, Serialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode)]
pub struct ConcurrencySettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub concurrency_key: Option<String>,
@@ -105,7 +103,65 @@ pub struct ConcurrencySettings {
pub concurrency_time_window_s: Option<i32>,
}
#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)]
/// Shared normalization for the positive-only `Option<i32>` runnable settings
/// (`concurrent_limit`, `timeout`, ...): a `<= 0` value is never meaningful — zero
/// concurrent slots permanently blocks a runnable at the concurrency gate (a re-queue
/// storm the zombie monitor eventually fails with a misleading OOM error), and a
/// 0-second timeout kills every job on the spot. The frontend already treats `0` as
/// "disabled", so `<= 0` maps to `None` (unset) everywhere. Idempotent.
pub fn none_if_non_positive(v: Option<i32>) -> Option<i32> {
v.filter(|n| *n > 0)
}
/// Coerce a `concurrent_limit <= 0` to disabled, dropping the now-meaningless time window
/// alongside it. Idempotent.
fn normalize_concurrency(
concurrent_limit: &mut Option<i32>,
concurrency_time_window_s: &mut Option<i32>,
) {
if none_if_non_positive(*concurrent_limit).is_none() {
*concurrent_limit = None;
*concurrency_time_window_s = None;
}
}
impl ConcurrencySettings {
pub fn normalized(mut self) -> Self {
normalize_concurrency(
&mut self.concurrent_limit,
&mut self.concurrency_time_window_s,
);
self
}
}
// Manual `Deserialize` so every ingestion path (script/flow create & update, app and
// http-trigger payloads, and read-back of already-stored settings) normalizes a `<= 0`
// limit uniformly, without each call site remembering to call `normalized()`.
impl<'de> Deserialize<'de> for ConcurrencySettings {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
struct Raw {
#[serde(default)]
concurrency_key: Option<String>,
#[serde(default)]
concurrent_limit: Option<i32>,
#[serde(default)]
concurrency_time_window_s: Option<i32>,
}
let Raw { concurrency_key, concurrent_limit, concurrency_time_window_s } =
Raw::deserialize(deserializer)?;
Ok(
ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }
.normalized(),
)
}
}
#[derive(Debug, Clone, Serialize, sqlx::FromRow, Default)]
pub struct ConcurrencySettingsWithCustom {
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_concurrency_key: Option<String>,
@@ -115,6 +171,41 @@ pub struct ConcurrencySettingsWithCustom {
pub concurrency_time_window_s: Option<i32>,
}
impl ConcurrencySettingsWithCustom {
pub fn normalized(mut self) -> Self {
normalize_concurrency(
&mut self.concurrent_limit,
&mut self.concurrency_time_window_s,
);
self
}
}
impl<'de> Deserialize<'de> for ConcurrencySettingsWithCustom {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
struct Raw {
#[serde(default)]
custom_concurrency_key: Option<String>,
#[serde(default)]
concurrent_limit: Option<i32>,
#[serde(default)]
concurrency_time_window_s: Option<i32>,
}
let Raw { custom_concurrency_key, concurrent_limit, concurrency_time_window_s } =
Raw::deserialize(deserializer)?;
Ok(ConcurrencySettingsWithCustom {
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
}
.normalized())
}
}
impl DebouncingSettings {
pub fn maybe_fallback(
self,
@@ -142,11 +233,15 @@ impl ConcurrencySettings {
concurrent_limit: Option<i32>,
concurrency_time_window_s: Option<i32>,
) -> Self {
// Legacy columns can still hold a stored `0` that predates ingestion normalization,
// so re-normalize here: this is the single load boundary for every DB-backed read
// (script/schedule read, flow value, and the worker pull path).
Self {
concurrency_key: self.concurrency_key.or(concurrency_key),
concurrent_limit: self.concurrent_limit.or(concurrent_limit),
concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s),
}
.normalized()
}
}
@@ -229,4 +324,91 @@ mod tests {
assert_eq!(r, Retry::default());
assert_eq!(r.exponential.multiplier, 1);
}
// The positive-only settings share one rule: `<= 0` means "unset". This is what keeps a
// stored `0` from being enforced as a zero-slot cap or a 0-second timeout.
#[test]
fn none_if_non_positive_coerces_zero_and_negative() {
assert_eq!(none_if_non_positive(Some(0)), None);
assert_eq!(none_if_non_positive(Some(-3)), None);
assert_eq!(none_if_non_positive(Some(1)), Some(1));
assert_eq!(none_if_non_positive(Some(i32::MAX)), Some(i32::MAX));
assert_eq!(none_if_non_positive(None), None);
}
// Ingestion path (scripts flatten this on `NewScript`, flows on `FlowModule`): a `0`
// concurrent_limit deserializes to disabled and drops the now-meaningless time window,
// while a real limit and its window survive untouched.
#[test]
fn concurrency_settings_deserialize_normalizes_non_positive_limit() {
let zero: ConcurrencySettings = serde_json::from_value(
serde_json::json!({"concurrent_limit": 0, "concurrency_time_window_s": 30}),
)
.unwrap();
assert_eq!(zero.concurrent_limit, None);
assert_eq!(zero.concurrency_time_window_s, None);
let negative: ConcurrencySettings =
serde_json::from_value(serde_json::json!({"concurrent_limit": -1})).unwrap();
assert_eq!(negative.concurrent_limit, None);
let real: ConcurrencySettings = serde_json::from_value(
serde_json::json!({"concurrent_limit": 2, "concurrency_time_window_s": 30}),
)
.unwrap();
assert_eq!(real.concurrent_limit, Some(2));
assert_eq!(real.concurrency_time_window_s, Some(30));
}
// Per-flow-step overrides use the `custom_concurrency_key` variant; same rule.
#[test]
fn concurrency_settings_with_custom_deserialize_normalizes() {
let zero: ConcurrencySettingsWithCustom = serde_json::from_value(
serde_json::json!({"concurrent_limit": 0, "concurrency_time_window_s": 5}),
)
.unwrap();
assert_eq!(zero.concurrent_limit, None);
assert_eq!(zero.concurrency_time_window_s, None);
}
// A normalized value serializes with the limit omitted (skip_serializing_if), matching the
// frontend's "disabled" representation instead of re-emitting a `0`.
#[test]
fn normalized_disabled_limit_serializes_as_omitted() {
let s =
ConcurrencySettings { concurrent_limit: Some(0), ..Default::default() }.normalized();
let json = serde_json::to_value(&s).unwrap();
assert!(json.get("concurrent_limit").is_none());
}
// Runtime load boundary: legacy rows still hold a raw `0` in the fallback columns. The
// fallback must not resurrect it as an active limit.
#[test]
fn maybe_fallback_normalizes_legacy_zero_column() {
let merged = ConcurrencySettings::default().maybe_fallback(None, Some(0), Some(30));
assert_eq!(merged.concurrent_limit, None);
assert_eq!(merged.concurrency_time_window_s, None);
}
// `NewScript`/`FlowModule` embed the settings via `#[serde(flatten)]`, which drives the
// manual Deserialize through a content-buffer deserializer rather than a plain map. Guard
// that path: normalization must still fire and sibling fields must still parse.
#[test]
fn flattened_concurrency_normalizes_and_preserves_siblings() {
#[derive(Deserialize)]
struct Wrapper {
name: String,
#[serde(flatten)]
concurrency: ConcurrencySettings,
}
let w: Wrapper = serde_json::from_value(serde_json::json!({
"name": "s",
"concurrent_limit": 0,
"concurrency_time_window_s": 42,
}))
.unwrap();
assert_eq!(w.name, "s");
assert_eq!(w.concurrency.concurrent_limit, None);
assert_eq!(w.concurrency.concurrency_time_window_s, None);
}
}
backend/windmill-worker/Cargo.toml
+4 -2
View File
@@ -115,9 +115,11 @@ hmac.workspace = true
pem = { workspace = true, optional = true }
rsa = { workspace = true, optional = true }
urlencoding.workspace = true
# `fs` adds flock(2) for the cross-process Python install lock (shared cache mounts);
# `user` adds geteuid(2) to verify ownership of the ansible socket-dir root
nix = { workspace = true, features = ["fs", "user"] }
nix = { workspace = true, features = ["user"] }
# Cross-platform advisory file lock (flock on unix, LockFileEx on windows) for the
# cross-process Python install lock into shared wheel-cache dirs.
fs4 = { workspace = true }
bytes.workspace = true
reqwest.workspace = true
reqwest-middleware.workspace = true
backend/windmill-worker/nsjail/run.docker.config.proto
+7
View File
@@ -77,6 +77,13 @@ mount {
is_bind: true
}
mount {
dst: "/dev/shm"
fstype: "tmpfs"
rw: true
is_bind: false
}
# Host DNS config layered over the image's /etc so name resolution works on the
# job's network (mandatory:false: some minimal images have no /etc files to shadow).
mount {
backend/windmill-worker/src/ai_executor.rs
+7 -4
View File
@@ -26,7 +26,7 @@ use windmill_ai::{
providers::create_query_builder,
query_builder::{BuildRequestArgs, ParsedResponse},
types::*,
utils::{should_use_structured_output_tool, AI_HTTP_CLIENT, AI_HTTP_HEADERS},
utils::{pinned_ai_client_for, should_use_structured_output_tool, AI_HTTP_HEADERS},
};
use windmill_common::{
cache,
@@ -984,11 +984,14 @@ pub async fn run_agent(
let resource_headers = &credentials.custom_headers;
// `endpoint` derives from the user-controlled provider base_url, so pin
// DNS to the SSRF-validated address: the connect must not rebind to an
// internal IP between the check and the request (TOCTOU).
let pinned_ai_client = pinned_ai_client_for(base_url).await?;
// Helper to build HTTP request with headers
let build_http_request = |body: String| {
// `endpoint` derives from the user-controlled provider base_url: use
// AI_HTTP_CLIENT, not the shared HTTP_CLIENT. See AI_HTTP_CLIENT.
let mut req = AI_HTTP_CLIENT
let mut req = pinned_ai_client
.post(&endpoint)
.timeout(timeout)
.header("Content-Type", "application/json");
backend/windmill-worker/src/common.rs
+3 -1
View File
@@ -1081,7 +1081,9 @@ pub async fn resolve_job_timeout(
*MAX_TIMEOUT_DURATION
};
match custom_timeout_secs {
// A `custom_timeout_secs <= 0` is not a 0-second limit but "unset": fall through to the
// default/global-max timeout instead of killing the job immediately.
match windmill_common::runnable_settings::none_if_non_positive(custom_timeout_secs) {
Some(timeout_secs)
if Duration::from_secs(timeout_secs as u64) < global_max_timeout_duration =>
{
backend/windmill-worker/src/duckdb_executor.rs
+103 -4
View File
@@ -2586,15 +2586,57 @@ async fn transform_attach_datatable(
.await?
}
};
let db_type = "postgres";
if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) {
hidden_passwords.lock().unwrap().push(pwd.to_string());
}
Ok(Some(
db_resource_to_attach_statements(db_resource, alias_name, db_type, None).await?,
))
Ok(Some(pg_secret_attach_statements(db_resource, alias_name)?))
}
// Secret names must be plain identifiers; the hash keeps two aliases distinct even
// when sanitizing maps them to the same string.
fn datatable_secret_name(alias: &str) -> String {
use sha2::{Digest, Sha256};
let sanitized: String = alias
.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
.collect();
let hash = &Sha256::digest(alias.as_bytes())[..4];
format!(
"__wm_datatable_{sanitized}_{:08x}",
u32::from_be_bytes([hash[0], hash[1], hash[2], hash[3]])
)
}
/// ATTACH a datatable's postgres database through a DuckDB TEMPORARY SECRET holding
/// the connection parameters; only sslmode rides in the ATTACH string.
fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result<Vec<String>> {
let res: PgDatabase = serde_json::from_value(db_resource)?;
// Escape single quotes: each field is embedded in a single-quoted DuckDB literal,
// so an unescaped quote would break out of the CREATE SECRET statement.
let esc = |s: &str| s.replace('\'', "''");
// The postgres secret type has no sslmode parameter, so it goes in the ATTACH
// string; only the libpq values PgDatabase::to_uri collapses to are forwarded.
let sslmode = match res.sslmode.as_deref() {
Some("disable") => "disable",
Some("require") | Some("verify-ca") | Some("verify-full") => "require",
_ => "prefer",
};
let secret_name = datatable_secret_name(alias_name);
Ok(vec![
"INSTALL postgres;".to_string(),
"LOAD postgres;".to_string(),
format!(
"CREATE OR REPLACE TEMPORARY SECRET {secret_name} (TYPE postgres, HOST '{}', PORT {}, DATABASE '{}', USER '{}', PASSWORD '{}');",
esc(&res.host),
res.port.unwrap_or(5432),
esc(&res.dbname),
esc(res.user.as_deref().unwrap_or("postgres")),
esc(res.password.as_deref().unwrap_or("")),
),
format!("ATTACH 'sslmode={sslmode}' AS {alias_name} (TYPE postgres, SECRET {secret_name});"),
])
}
async fn transform_s3_uris(query: &str) -> Result<String> {
@@ -3731,6 +3773,63 @@ mod tests {
assert!(result.contains("sslmode=prefer"));
}
#[test]
fn test_pg_secret_attach_statements() {
let db_resource = json!({
"host": "localhost",
"port": 5433,
"user": "custom_instance_user",
"password": "it's-secret",
"dbname": "wm_datatables",
"sslmode": "require"
});
let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap();
assert_eq!(stmts[0], "INSTALL postgres;");
assert_eq!(stmts[1], "LOAD postgres;");
let secret_name = datatable_secret_name("dt");
assert_eq!(
stmts[2],
format!(
"CREATE OR REPLACE TEMPORARY SECRET {secret_name} (TYPE postgres, HOST 'localhost', PORT 5433, DATABASE 'wm_datatables', USER 'custom_instance_user', PASSWORD 'it''s-secret');"
)
);
assert_eq!(
stmts[3],
format!("ATTACH 'sslmode=require' AS dt (TYPE postgres, SECRET {secret_name});")
);
}
#[test]
fn test_pg_secret_attach_statements_sslmode_whitelist() {
for (input, expected) in [
(Some("allow"), "prefer"),
(Some("verify-full"), "require"),
(Some("disable"), "disable"),
(Some("unknown-value"), "prefer"),
(None, "prefer"),
] {
let mut db_resource = json!({ "host": "h", "dbname": "d" });
if let Some(s) = input {
db_resource["sslmode"] = json!(s);
}
let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap();
assert!(
stmts[3].starts_with(&format!("ATTACH 'sslmode={expected}'")),
"sslmode {input:?} → {}",
stmts[3]
);
}
}
#[test]
fn test_datatable_secret_name_sanitizes_and_disambiguates() {
let a = datatable_secret_name("a.b");
let b = datatable_secret_name("a_b");
assert!(a.starts_with("__wm_datatable_a_b_"));
assert_ne!(a, b);
assert!(a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'));
}
#[test]
fn test_format_attach_db_conn_str_bigquery() {
let db_resource = json!({
backend/windmill-worker/src/python_executor.rs
+28 -23
View File
@@ -2832,11 +2832,12 @@ pub async fn handle_python_reqs(
};
// Cross-process advisory lock. Best-effort: if the filesystem doesn't
// support flock we log and proceed — verify_wheel_record + job retry
// still guard correctness, just without the dedup.
#[cfg(unix)]
// support locking we log and proceed — verify_wheel_record + job retry
// still guard correctness, just without the dedup. Cross-platform
// (flock on unix, LockFileEx on windows) so agents sharing a wheel-cache
// dir on a Windows host serialize just as they do on unix.
let _venv_file_lock: Option<std::fs::File> = {
use std::os::unix::io::AsRawFd;
use fs4::fs_std::FileExt;
let lock_path = format!("{venv_p}.lock");
if let Some(parent) = std::path::Path::new(&lock_path).parent() {
let _ = std::fs::create_dir_all(parent);
@@ -2844,17 +2845,17 @@ pub async fn handle_python_reqs(
match std::fs::OpenOptions::new().create(true).write(true).open(&lock_path) {
Ok(f) => {
// Bounded wait: a holder that crashes releases the lock (the
// kernel drops it on fd close), but a live-but-stuck holder
// OS drops it on handle close), but a live-but-stuck holder
// (e.g. uv wedged on a hung mount) would otherwise block us
// forever. After the cap, proceed degraded rather than hang —
// verify_wheel_record + retry still guard correctness.
const MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(300);
let waited_since = std::time::Instant::now();
loop {
match nix::fcntl::flock(f.as_raw_fd(), nix::fcntl::FlockArg::LockExclusiveNonblock) {
Ok(()) => break Some(f),
// EWOULDBLOCK == EAGAIN on Linux: another holder has the lock.
Err(nix::errno::Errno::EWOULDBLOCK) => {
match f.try_lock_exclusive() {
Ok(true) => break Some(f),
// Another holder has the lock.
Ok(false) => {
if waited_since.elapsed() >= MAX_WAIT {
tracing::warn!(
workspace_id = %w_id,
@@ -2876,7 +2877,7 @@ pub async fn handle_python_reqs(
Err(e) => {
tracing::warn!(
workspace_id = %w_id,
"could not flock {lock_path}, proceeding without cross-process install lock: {e}"
"could not lock {lock_path}, proceeding without cross-process install lock: {e}"
);
break Some(f);
}
@@ -3782,14 +3783,14 @@ mod tests {
);
}
#[cfg(unix)]
#[tokio::test]
async fn test_venv_file_lock_excludes_across_descriptions() {
// The cross-process layer: flock on a sibling `.lock` excludes a second
// independent open file description (i.e. another worker process) while
// held, and frees it on close. Mirrors the loop in handle_python_reqs.
use nix::fcntl::{flock, FlockArg};
use std::os::unix::io::AsRawFd;
// The cross-process layer: an advisory lock on a sibling `.lock` excludes a
// second independent open file handle (i.e. another worker process) while
// held, and frees it on close. Mirrors the loop in handle_python_reqs and
// must hold on every platform (flock on unix, LockFileEx on windows) — a
// Windows host running several agents against one wheel cache relies on it.
use fs4::fs_std::FileExt;
let dir = std::env::temp_dir().join("wm_venv_lock_test");
std::fs::create_dir_all(&dir).unwrap();
@@ -3800,24 +3801,28 @@ mod tests {
.write(true)
.open(&lock_path)
.unwrap();
flock(f1.as_raw_fd(), FlockArg::LockExclusiveNonblock).unwrap();
assert!(
f1.try_lock_exclusive().unwrap(),
"first holder must acquire the lock"
);
// A second descriptor (stand-in for another process) cannot take it.
// A second handle (stand-in for another process) cannot take it.
let f2 = std::fs::OpenOptions::new()
.create(true)
.write(true)
.open(&lock_path)
.unwrap();
assert_eq!(
flock(f2.as_raw_fd(), FlockArg::LockExclusiveNonblock),
Err(nix::errno::Errno::EWOULDBLOCK),
assert!(
!f2.try_lock_exclusive().unwrap(),
"a second holder must be blocked while the lock is held"
);
// Releasing the first lets the second acquire it.
drop(f1);
flock(f2.as_raw_fd(), FlockArg::LockExclusiveNonblock)
.expect("lock must be acquirable once the holder releases it");
assert!(
f2.try_lock_exclusive().unwrap(),
"lock must be acquirable once the holder releases it"
);
drop(f2);
let _ = std::fs::remove_file(&lock_path);
backend/windmill-worker/src/result_processor.rs
+121 -23
View File
@@ -908,8 +908,10 @@ async fn maybe_reconcile_git_sync_auto_pull(
/// derivation: a dev workspace deploys to its environment-label branch
/// (`dev`/`staging`), other fork workspaces to `wm-fork/<base>/<id-suffix>`,
/// else the promotion `wm_deploy/**` formula (per-folder or per-item form).
/// `None` when the deploy stays on the base branch (workspace-wide mode) and
/// has no PR to open.
/// A dev workspace in promotion mode is the exception: it takes the promotion
/// `wm_deploy/**` formula (per-item PRs into the parent) instead of its label
/// branch. `None` when the deploy stays on the base branch (workspace-wide
/// mode) and has no PR to open.
#[cfg(all(feature = "enterprise", feature = "private"))]
fn git_sync_deploy_pr_head_branch(
workspace_id: &str,
@@ -922,18 +924,23 @@ fn git_sync_deploy_pr_head_branch(
item_parent_path: &str,
path_type: &str,
) -> Option<String> {
if dev_workspace_label.is_some() {
return Some(windmill_common::workspaces::dev_workspace_branch(
dev_workspace_label,
));
}
let is_fork = parent_workspace_id.is_some()
|| workspace_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX);
if is_fork {
let suffix = workspace_id
.strip_prefix(windmill_common::workspaces::WM_FORK_PREFIX)
.unwrap_or(workspace_id);
return Some(format!("wm-fork/{base}/{suffix}"));
let is_dev = dev_workspace_label.is_some();
// A dev workspace with promotion on falls through to the wm_deploy/**
// formula below; the label/fork branches only apply when promotion is off.
if !(is_dev && use_individual_branch) {
if is_dev {
return Some(windmill_common::workspaces::dev_workspace_branch(
dev_workspace_label,
));
}
let is_fork = parent_workspace_id.is_some()
|| workspace_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX);
if is_fork {
let suffix = workspace_id
.strip_prefix(windmill_common::workspaces::WM_FORK_PREFIX)
.unwrap_or(workspace_id);
return Some(format!("wm-fork/{base}/{suffix}"));
}
}
if !use_individual_branch {
return None;
@@ -1074,14 +1081,19 @@ async fn maybe_open_git_sync_deploy_pr(
return;
};
let repo_url =
match windmill_common::git_sync_ee::resolve_repo_url(db, workspace_id, &repo_path).await {
Ok(url) => url,
Err(e) => {
tracing::warn!("git sync PR: could not resolve repo url for {repo_path}: {e:#}");
return;
}
};
let repo_url = match windmill_common::git_sync_ee::resolve_repo_url_interpolated(
db,
workspace_id,
&repo_path,
)
.await
{
Ok(url) => url,
Err(e) => {
tracing::warn!("git sync PR: could not resolve repo url for {repo_path}: {e:#}");
return;
}
};
// A fork of a dev workspace diverged from the dev's label branch, so its PR
// merges back there; everything else targets the tracked branch.
let pr_base = row.parent_dev_workspace_label.as_deref().unwrap_or(&base);
@@ -1245,9 +1257,22 @@ async fn maybe_post_git_sync_check(
(None, Some(deploy)) => (true, deploy),
(None, None) => return,
};
let Ok(check) = serde_json::from_value::<GitSyncCheck>(marker) else {
let Ok(mut check) = serde_json::from_value::<GitSyncCheck>(marker) else {
return;
};
// Markers carry the literal resource URL (job args are persisted, so a
// `$var:`-resolved URL must not land there); interpolate before calling
// GitHub.
check.repo_url =
match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id)
.await
{
Ok(u) => u,
Err(e) => {
tracing::error!("git sync-check: cannot interpolate repo url: {e:#}");
return;
}
};
// "In sync" on a PR that visibly changes files reads as a bug when those
// files are outside the repo's sync filters — say what the scope is.
let scope_note = if !is_deploy && success {
@@ -2220,4 +2245,77 @@ mod git_sync_pr_tests {
Some("dev".to_string())
);
}
#[test]
fn dev_workspace_promotion_uses_wm_deploy_branch() {
// Promotion on: a dev workspace gets per-item wm_deploy/** branches
// (namespaced by its own id), not its env-label branch.
assert_eq!(
git_sync_deploy_pr_head_branch(
"dev",
Some("prod"),
Some("dev"),
"main",
true,
false,
"f/folder/my_script",
"",
"script"
),
Some("wm_deploy/dev/script/f__folder__my_script".to_string())
);
// Per-folder form still honored for a promotion dev workspace.
assert_eq!(
git_sync_deploy_pr_head_branch(
"dev",
Some("prod"),
Some("dev"),
"main",
true,
true,
"f/folder/my_script",
"",
"script"
),
Some("wm_deploy/dev/f__folder".to_string())
);
// Promotion off: the env-label branch still wins.
assert_eq!(
git_sync_deploy_pr_head_branch(
"dev",
Some("prod"),
Some("dev"),
"main",
false,
false,
"f/x/y",
"",
"script"
),
Some("dev".to_string())
);
}
#[test]
fn dev_promotion_user_group_items_open_no_pr() {
// User/group objects get no wm_deploy branch even on a dev workspace; the
// CLI isolates them to the env-label branch, so the backend opens no PR
// (never a PR from the env-label branch into the parent for these).
for path_type in ["user", "group"] {
assert_eq!(
git_sync_deploy_pr_head_branch(
"dev",
Some("prod"),
Some("dev"),
"main",
true,
false,
"u/alice",
"",
path_type
),
None
);
}
}
}
backend/windmill-worker/src/worker.rs
+10 -9
View File
@@ -3892,15 +3892,16 @@ pub async fn handle_queued_job(
#[cfg(not(feature = "enterprise"))]
if let Connection::Sql(db) = conn {
if (job.concurrent_limit.is_some()
|| windmill_common::runnable_settings::prefetch_cached_from_handle(
job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit
.is_some())
if (windmill_queue::jobs::has_active_concurrency_limit(job.concurrent_limit)
|| windmill_queue::jobs::has_active_concurrency_limit(
windmill_common::runnable_settings::prefetch_cached_from_handle(
job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit,
))
&& !job.kind.is_dependency()
{
logs.push_str("---\n");
backend/windmill-worker/src/worker_flow.rs
+58 -20
View File
@@ -1525,9 +1525,14 @@ pub async fn update_flow_status_after_job_completion_internal(
let concurrency_key = tag_and_concurrency_key
.as_ref()
.and_then(|x| x.concurrency_key.clone());
let concurrent_limit = tag_and_concurrency_key
.as_ref()
.and_then(|x| x.concurrent_limit);
// `concurrent_limit` here can come straight from the raw flow JSON (see
// get_tag_and_concurrency), bypassing the ConcurrencySettings deserialization guard,
// so a stored `0` must still be coerced to disabled before we register a key for it.
let concurrent_limit = windmill_common::runnable_settings::none_if_non_positive(
tag_and_concurrency_key
.as_ref()
.and_then(|x| x.concurrent_limit),
);
let concurrency_time_window_s = tag_and_concurrency_key
.as_ref()
.and_then(|x| x.concurrency_time_window_s);
@@ -3398,10 +3403,16 @@ async fn push_next_flow_job(
// Persist approval user groups conditions, if any. Requires runnning the InputTransform
let required_events = suspend.required_events.unwrap() as u16;
let user_auth_required = suspend.user_auth_required.unwrap_or(false);
if user_auth_required {
let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false);
let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false);
// self_approval_disabled must be persisted even without user_auth_required, otherwise
// the resume boundary sees no approval_conditions and the restriction is silently
// dropped. user_groups_required only applies together with user_auth_required.
if user_auth_required || self_approval_disabled {
let user_groups_required: Vec<String>;
if let Some(user_groups_required_as_input_transform) = suspend.user_groups_required
if !user_auth_required {
user_groups_required = Vec::new();
} else if let Some(user_groups_required_as_input_transform) =
suspend.user_groups_required
{
match user_groups_required_as_input_transform {
InputTransform::Static { value } => {
@@ -3566,7 +3577,7 @@ async fn push_next_flow_job(
count: required_events,
job: last
}),
(required_events - resume_messages.len() as u16) as i32,
(required_events.saturating_sub(resume_messages.len() as u16)) as i32,
Duration::from_secs(
suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60)
) as Duration,
@@ -4383,13 +4394,10 @@ async fn push_next_flow_job(
)
.await?;
if timeout_value < 0 {
return Err(Error::ExecutionErr(
"Timeout value cannot be negative".to_string(),
));
}
Some(timeout_value)
// A `<= 0` step timeout (including a negative eval) means "no override": fall back
// to the referenced runnable's own timeout rather than a 0-second/negative timeout
// that would kill the step instantly.
effective_flow_step_timeout(Some(timeout_value), payload_tag.timeout)
} else {
payload_tag.timeout
};
@@ -6042,6 +6050,18 @@ async fn flow_to_payload(
})
}
/// Effective timeout for a flow step given the module's (already-evaluated) timeout override and
/// the timeout inherited from the referenced runnable. A `<= 0` override — or none — means "no
/// override": fall back to the inherited value (which is itself `None` when unset, i.e. the
/// instance default). A positive override wins. This keeps a step `timeout: 0` equivalent to an
/// omitted one rather than a 0-second, instant-kill timeout.
pub(crate) fn effective_flow_step_timeout(
module_override: Option<i32>,
inherited: Option<i32>,
) -> Option<i32> {
windmill_common::runnable_settings::none_if_non_positive(module_override).or(inherited)
}
pub async fn script_to_payload(
script_hash: Option<windmill_common::scripts::ScriptHash>,
script_path: String,
@@ -6131,11 +6151,13 @@ pub async fn script_to_payload(
module.delete_after_use.unwrap_or(false) || delete_after_use.unwrap_or(false);
let final_delete_after_secs = module.delete_after_secs.or(delete_after_secs);
let flow_step_timeout = if module.timeout.is_some() {
None
} else {
script_timeout
};
// Always carry the referenced script's own timeout as the inherited fallback. The module's
// timeout override (if any) is selected at the push site, where a `<= 0` override is treated
// as "no override" and falls back to this value — so `timeout: 0` on a step means "use the
// script's timeout", not a 0-second (immediate-kill) timeout. Normalize the inherited value
// too, so a legacy `0` script timeout resolves to the default rather than a zero-second kill.
let flow_step_timeout =
windmill_common::runnable_settings::none_if_non_positive(script_timeout);
Ok(JobPayloadWithTag {
payload,
tag,
@@ -6260,9 +6282,25 @@ pub async fn get_previous_job_result(
#[cfg(test)]
mod tests {
use super::extract_chat_message_from_flow_result;
use super::{effective_flow_step_timeout, extract_chat_message_from_flow_result};
use serde_json::{json, value::to_raw_value};
// A `<= 0` step timeout override must behave as "no override" and inherit the referenced
// script's timeout, not collapse to a 0-second (instant-kill) timeout. A positive override
// still wins. Guards the flow-step timeout footgun.
#[test]
fn flow_step_timeout_zero_or_negative_inherits_script_timeout() {
// zero / negative override -> inherited script timeout
assert_eq!(effective_flow_step_timeout(Some(0), Some(300)), Some(300));
assert_eq!(effective_flow_step_timeout(Some(-5), Some(300)), Some(300));
// no inherited timeout either -> None (falls through to the instance default)
assert_eq!(effective_flow_step_timeout(Some(0), None), None);
// positive override wins over the inherited value
assert_eq!(effective_flow_step_timeout(Some(120), Some(300)), Some(120));
// no override -> inherited
assert_eq!(effective_flow_step_timeout(None, Some(300)), Some(300));
}
#[test]
fn pretty_prints_full_result_when_no_override_is_present() {
let value = json!({
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.764.0";
export const VERSION = "v1.769.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/bun.lock
+4 -4
View File
@@ -19,7 +19,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.749.0",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
@@ -28,7 +28,7 @@
"windmill-parser-wasm-py": "1.693.1",
"windmill-parser-wasm-py-imports": "1.693.1",
"windmill-parser-wasm-r": "1.668.1",
"windmill-parser-wasm-regex": "1.692.0",
"windmill-parser-wasm-regex": "1.764.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.695.0",
@@ -290,7 +290,7 @@
"utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="],
"windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.749.0", "", {}, "sha512-gj8g9sWQ0tXKfXso7xJxR56sS8Loe/RsnFy+0af5R8siZeCag9ikquGbQ8d8kOqIK2U8eCNA0LqsU/xAFDJIOg=="],
"windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.753.0", "", {}, "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="],
"windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="],
@@ -308,7 +308,7 @@
"windmill-parser-wasm-r": ["windmill-parser-wasm-r@1.668.1", "", {}, "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ=="],
"windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.692.0", "", {}, "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw=="],
"windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.764.0", "", {}, "sha512-V2eFdKD90gqWikOvjl2fwMpFqiFt/21+4iQMbiNJYl7Lm2UiEcEZ4r9bpgJLG4TLOLqvD6+u4Ju3WaytxN2O2w=="],
"windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="],
cli/package-lock.json
Generated
+4 -4
View File
@@ -20,7 +20,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.749.0",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
@@ -1413,9 +1413,9 @@
}
},
"node_modules/windmill-parser-wasm-asset": {
"version": "1.749.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.749.0.tgz",
"integrity": "sha512-gj8g9sWQ0tXKfXso7xJxR56sS8Loe/RsnFy+0af5R8siZeCag9ikquGbQ8d8kOqIK2U8eCNA0LqsU/xAFDJIOg=="
"version": "1.753.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.753.0.tgz",
"integrity": "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="
},
"node_modules/windmill-parser-wasm-csharp": {
"version": "1.510.1",
cli/package.json
+2 -2
View File
@@ -28,7 +28,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.749.0",
"windmill-parser-wasm-asset": "1.753.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.761.0",
"windmill-parser-wasm-java": "1.510.1",
@@ -54,4 +54,4 @@
"@types/ws": "^8.5.0",
"typescript": "^5.7.0"
}
}
}
cli/src/commands/app/bundle.ts
+18 -1
View File
@@ -35,10 +35,24 @@ export const DEFAULT_BUILD_OPTIONS = {
loader: {
".css": "css" as const,
},
// esbuild export conditions safe for any app: "style" resolves tailwindcss v4's CSS
// entry (@import "tailwindcss"); "module" is re-added because esbuild drops its
// auto-included "module" default once any custom condition is set. The Svelte-only
// "svelte" condition is gated per-app in conditionsFor().
conditions: ["style", "module"],
logLevel: "info" as const,
write: true,
};
// "svelte" points at raw .svelte sources that only compile with the Svelte plugin, so
// enable it only for Svelte apps — for a plain app a Svelte-dual-published import would
// otherwise resolve to .svelte and hard-fail with no loader configured.
function conditionsFor(svelte: boolean): string[] {
return svelte
? [...DEFAULT_BUILD_OPTIONS.conditions, "svelte"]
: DEFAULT_BUILD_OPTIONS.conditions;
}
/**
* Detects which frontend frameworks are present in package.json
*/
@@ -284,6 +298,7 @@ export async function createBundle(
const buildOptions = {
...DEFAULT_BUILD_OPTIONS,
conditions: conditionsFor(frameworks.svelte),
entryPoints: [entryPoint],
outfile,
sourcemap,
@@ -337,11 +352,13 @@ export async function createBundle(
/**
* Gets the esbuild build options for use in watch mode (dev server)
* @param entryPoint Entry point file
* @param svelte Whether the app is a Svelte app (enables the "svelte" condition)
* @returns esbuild build options
*/
export function getDevBuildOptions(entryPoint: string = "index.tsx") {
export function getDevBuildOptions(entryPoint: string = "index.tsx", svelte = false) {
return {
...DEFAULT_BUILD_OPTIONS,
conditions: conditionsFor(svelte),
entryPoints: [entryPoint],
outfile: "dist/bundle.js",
sourcemap: true,
cli/src/commands/app/dev.ts
+1 -1
View File
@@ -538,7 +538,7 @@ async function dev(opts: DevOptions, appFolder?: string) {
});
}
const buildOptions = getDevBuildOptions(entryPoint);
const buildOptions = getDevBuildOptions(entryPoint, frameworks.svelte);
// Load framework-specific plugins (svelte, vue) based on package.json
const frameworkPlugins = await createFrameworkPlugins(appDir);
cli/src/commands/pipeline/boundedCascade.ts
+4 -6
View File
@@ -87,12 +87,10 @@ export function assetUriToNodeId(uri: string): string | undefined {
if (!m) return undefined;
const prefix = m[1].toLowerCase();
const kind = prefix === "s3" ? "s3object" : prefix;
// Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys so a
// `--to s3:///exports/x` token resolves to the canonical graph node
// `s3object:exports/x` (default storage), same as `s3://exports/x`, and a
// canonical key never starts with `/`.
const path = kind === "s3object" ? m[2].replace(/^\/+/, "") : m[2];
return `${kind}:${path}`;
// The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3
// path encodes the storage, with a leading `/` for the workspace default
// (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`.
return `${kind}:${m[2]}`;
}
export type LineageDag = {
cli/src/commands/pipeline/localGraph.ts
+8 -15
View File
@@ -300,13 +300,10 @@ function fallbackParse(content: string, language: string): ParseAssetsRaw {
if (uri) {
const prefix = uri[1].toLowerCase();
const kind = prefix === "s3" ? "s3object" : prefix;
// Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys
// so `s3:///key` (default storage) and `s3://key` / DuckDB canonicalize
// to the same node id (and a canonical key never starts with `/`) —
// otherwise a go/bash fallback consumer's `// on s3:///x` would not
// connect to a wasm-inferred `x` producer.
const path = kind === "s3object" ? uri[2].replace(/^\/+/, "") : uri[2];
out.triggers!.push({ kind: "asset", asset_kind: kind, path });
// The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3
// path encodes the storage, with a leading `/` for the workspace default
// (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`.
out.triggers!.push({ kind: "asset", asset_kind: kind, path: uri[2] });
} else if (NATIVE_KINDS.has(firstTok) && rest === firstTok) {
// A native marker (`// on data_upload`) must stand alone: the canonical
// parser rejects a marker line with trailing content (`// on data_upload
@@ -393,14 +390,10 @@ export function parseMuteAnnotations(content: string): {
}
for (const [prefix, kind] of MUTE_ASSET_PREFIXES) {
if (arg.startsWith(prefix)) {
// S3 canonicalization as in `parse_asset_syntax`: strip every leading
// slash so `s3:///key` (default storage) mutes the same node as the
// inferred bare `key`.
const p =
kind === "s3object"
? arg.slice(prefix.length).replace(/^\/+/, "")
: arg.slice(prefix.length);
muted.add(`${kind}:${p}`);
// The suffix is kept verbatim, as in `parse_asset_syntax` — a muted
// `s3:///key` (default storage, path `/key`) only matches an inferred
// default-storage read of the same object.
muted.add(`${kind}:${arg.slice(prefix.length)}`);
break;
}
}
cli/src/commands/script/script.ts
+44 -7
View File
@@ -100,6 +100,30 @@ export function isRawAppBackendPath(filePath: string): boolean {
return isRawAppBackendPathInternal(filePath);
}
/**
* The positive-only runnable settings (concurrent_limit, timeout, ...) treat any `<= 0`
* value as "unset": the backend coerces it to null (a 0-slot concurrency limit bricks the
* runnable, a 0s timeout kills every run). Coerce to undefined so it is serialized as
* omitted, never as 0, and redeploys don't churn against the backend-normalized value.
*/
export function nonePositiveInt(
v: number | undefined | null
): number | undefined {
return v != null && v > 0 ? v : undefined;
}
/**
* Normalize a concurrent_limit + its time window together: when the limit is disabled
* (<= 0) the window is dropped too. Returns [concurrent_limit, concurrency_time_window_s].
*/
export function normalizeConcurrency(
concurrentLimit: number | undefined | null,
concurrencyTimeWindowS?: number | undefined | null
): [number | undefined, number | undefined] {
const limit = nonePositiveInt(concurrentLimit);
return limit === undefined ? [undefined, undefined] : [limit, concurrencyTimeWindowS ?? undefined];
}
/**
* Checks if a path is inside a normal app folder (inline script).
* Matches patterns like: .../myApp.app/... or .../myApp__app/...
@@ -469,6 +493,15 @@ export async function handleFile(
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint);
// A concurrent_limit of <= 0 means "concurrency disabled", not "zero slots" (which
// would brick the runnable at the queue's concurrency gate). Emit it as omitted rather
// than 0 so a redeploy never re-persists a zero-slot limit, and drop the now-meaningless
// time window alongside it. Mirrors the backend's ConcurrencySettings::normalized.
const [normConcurrentLimit, normConcurrencyTimeWindowS] = normalizeConcurrency(
typed?.concurrent_limit,
typed?.concurrency_time_window_s
);
const requestBodyCommon: NewScript = {
content,
description: typed?.description ?? "",
@@ -482,8 +515,8 @@ export async function handleFile(
ws_error_handler_muted: typed?.ws_error_handler_muted,
dedicated_worker: typed?.dedicated_worker,
cache_ttl: typed?.cache_ttl,
concurrency_time_window_s: typed?.concurrency_time_window_s,
concurrent_limit: typed?.concurrent_limit,
concurrency_time_window_s: normConcurrencyTimeWindowS,
concurrent_limit: normConcurrentLimit,
deployment_message: message,
restart_unless_cancelled: typed?.restart_unless_cancelled,
visible_to_runner_only: typed?.visible_to_runner_only,
@@ -493,7 +526,7 @@ export async function handleFile(
debounce_key: typed?.debounce_key,
debounce_delay_s: typed?.debounce_delay_s,
codebase: await codebase?.getDigest(forceTar),
timeout: typed?.timeout,
timeout: nonePositiveInt(typed?.timeout),
on_behalf_of_email: typed?.on_behalf_of_email,
envs: typed?.envs,
modules: modules,
@@ -530,9 +563,13 @@ export async function handleFile(
remote.ws_error_handler_muted &&
typed.dedicated_worker == remote.dedicated_worker &&
typed.cache_ttl == remote.cache_ttl &&
typed.concurrency_time_window_s ==
remote.concurrency_time_window_s &&
typed.concurrent_limit == remote.concurrent_limit &&
normConcurrencyTimeWindowS ==
normalizeConcurrency(
remote.concurrent_limit,
remote.concurrency_time_window_s
)[1] &&
normConcurrentLimit ==
normalizeConcurrency(remote.concurrent_limit)[0] &&
Boolean(typed.restart_unless_cancelled) ==
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
@@ -540,7 +577,7 @@ export async function handleFile(
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout &&
nonePositiveInt(typed.timeout) == nonePositiveInt(remote.timeout) &&
//@ts-ignore
typed.concurrency_key == remote["concurrency_key"] &&
typed.debounce_key == remote["debounce_key"] &&
cli/src/commands/shared_ui.ts
+58 -15
View File
@@ -26,34 +26,77 @@ async function readDirRecursive(
return out;
}
export type SharedUiChange =
| { type: "added"; path: string }
| { type: "edited"; path: string; before: string; after: string }
| { type: "deleted"; path: string };
/**
* Diff the local <cwd>/ui/ folder against the workspace's shared UI store in
* the push direction (local -> remote), returning entries whose `path` is
* prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the
* dry-run preview and the real push never diverge.
*
* Mirrors pushSharedUi's no-op: with no local ui/ folder there is nothing to
* push, so the apply is a no-op and the preview must be empty (even when the
* remote store is non-empty) to avoid phantom diffs the apply won't perform.
*/
export async function diffSharedUi(workspace: string): Promise<SharedUiChange[]> {
const localDir = path.join(process.cwd(), SHARED_UI_DIR);
if (!fs.existsSync(localDir)) {
return [];
}
const files = await readDirRecursive(localDir);
let remote: Record<string, string> = {};
try {
const got = await wmill.getSharedUi({ workspace });
remote = got.files ?? {};
} catch {
// If endpoint missing or unauthorized, treat remote as empty (the push
// would attempt the PUT anyway).
}
// Use Object.hasOwn, not `in`: a file named after an Object.prototype member
// (e.g. ui/toString) would otherwise register as always-present and be
// misdiffed.
const changes: SharedUiChange[] = [];
for (const [rel, content] of Object.entries(files)) {
const p = `${SHARED_UI_DIR}/${rel}`;
if (!Object.hasOwn(remote, rel)) {
changes.push({ type: "added", path: p });
} else if (remote[rel] !== content) {
changes.push({ type: "edited", path: p, before: remote[rel], after: content });
}
}
for (const rel of Object.keys(remote)) {
if (!Object.hasOwn(files, rel)) {
changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` });
}
}
return changes;
}
/**
* Push the local <cwd>/ui/ folder to the workspace's shared UI store.
* Returns true if a push was performed, false if the folder is missing or empty.
* Returns true if a push was performed, false if the folder is missing or
* already matches the remote store. Note an empty-but-existing folder still
* pushes an empty map (clearing the remote store) if the remote is non-empty.
*/
export async function pushSharedUi(workspace: string): Promise<boolean> {
const localDir = path.join(process.cwd(), SHARED_UI_DIR);
if (!fs.existsSync(localDir)) {
return false;
}
const files = await readDirRecursive(localDir);
// Skip if no change
let remote: Record<string, string> = {};
try {
const got = await wmill.getSharedUi({ workspace });
remote = got.files ?? {};
} catch {
// If endpoint missing or unauthorized, just attempt the PUT
}
if (
Object.keys(remote).length === Object.keys(files).length &&
Object.entries(files).every(([k, v]) => remote[k] === v)
) {
// Skip if no change — reuse diffSharedUi so preview and push never diverge.
const diff = await diffSharedUi(workspace);
if (diff.length === 0) {
log.info(colors.gray("Shared UI folder up to date"));
return false;
}
const files = await readDirRecursive(localDir);
await wmill.updateSharedUi({
workspace,
requestBody: { files },

Some files were not shown because too many files have changed in this diff Show More