diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md index 61758550b2..bd40b6c472 100644 --- a/.agents/skills/pr/SKILL.md +++ b/.agents/skills/pr/SKILL.md @@ -1,18 +1,19 @@ --- name: pr user_invocable: true -description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. +description: Open a draft pull request on GitHub and drive CI review rounds until it is ready. MUST use when you want to create/open a PR. --- # Pull Request Skill -Create a draft pull request with a clear title and explicit description of changes. +Create a draft pull request with a clear title and explicit description of changes, then drive it through CI review rounds to ready. ## Instructions 1. **Analyze branch changes**: Understand all commits since diverging from main 2. **Push to remote**: Ensure all commits are pushed 3. **Create draft PR**: Always open as draft for review before merging +4. **Drive review rounds**: trigger CI reviews on the draft and only flip to ready once every verdict is a go (see "Review rounds" below) ## PR Title Format @@ -124,6 +125,42 @@ and continue once they confirm it's done. )" ``` 9. Return the PR URL to the user +10. Drive the PR through CI review rounds to ready (see "Review rounds" below) + +## Review rounds (draft → ready) + +A PR leaves draft **only after a clean CI review round**. Never run `gh pr ready` before that. + +1. **Trigger a round and wait for it**: launch the waiter as a background Bash task (a round takes 10–30 min; you are woken when it exits — do not stop the session or poll in the foreground while it runs): + + ```bash + bash .agents/skills/pr/review-round.sh + ``` + + It comments `/review` on the PR — which runs the Codex, Claude and Pi CI reviewers even on a draft — waits for the spawned `PR Review Commands` workflow run(s) to complete, then prints one verdict line per reviewer and saves the full review comments to files. + + `/review` (and `/codex`) are **idempotent per head SHA**: if a running or successful review already covers the current head, they skip that agent and post nothing new — the waiter reads the existing verdict for that head, so a skipped agent is *not* a missing one. A cancelled/failed head run is re-run in place; a fresh run is launched only when nothing covers the head. So an unchanged-head re-review is a near no-op, not a new round — push a commit to get genuinely fresh reviews. + +2. **Judge the round.** Codex is mandatory; Claude, Pi and cubic count whenever they posted. Every review starts with one of the three `REVIEW.md` verdicts: + - Codex verdict missing → the round is void: the waiter warns only when the head has no green Codex run (cancelled/failed/absent — not merely skipped-because-already-reviewed). Comment `/codex` on the PR, which re-runs the interrupted run in place (or launches one if none exists), wait the same way, and judge again. + - Any **"Should address issues before merging"** → fix the P0/P1 findings (and the nits while you're there), commit, push, and start a new round (step 1). + - Only **"Mergeable, but should ideally address nits"** and/or **"Good to merge"** → fix the nits too; a nit that is wrong or genuinely not worth fixing may instead be dismissed by replying to the review comment with your reasoning. Push nit-only fixes without starting another full round. + +3. **Flip to ready with the marker comment.** The review workflows skip the redundant `ready_for_review`-triggered round when the PR author has posted a marker naming the current head SHA **and** the PR's latest Codex review *posted before the marker* has a non-blocking verdict (reviewer evidence — a bare marker with no round behind it, or one whose last pre-marker Codex verdict is "Should address issues", skips nothing). Keep the prefix exact and use the full 40-char SHA of the head you are flipping: + - every verdict was "Good to merge" (head unchanged since the round): + + `✅ Review round clean @ ` + + - nit-only round, nits fixed or dismissed afterwards (head may have moved past the reviewed SHA — say so): + + `✅ Review round clean @ — nit-only verdicts at ; nits addressed in / dismissed in review replies` + + ```bash + gh pr comment --body "✅ Review round clean @ $(git rev-parse HEAD)" + gh pr ready + ``` + + If any P0/P1 finding is unaddressed or the head moved for reasons other than nit fixes, do **not** post the marker or flip — run another round instead. ## EE Companion PR (when `*_ee.rs` files were modified) diff --git a/.agents/skills/pr/review-round.sh b/.agents/skills/pr/review-round.sh new file mode 100755 index 0000000000..abd94005f6 --- /dev/null +++ b/.agents/skills/pr/review-round.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# Trigger a CI review round on a PR and wait for it to finish. +# +# Usage: bash .agents/skills/pr/review-round.sh [PR_NUMBER] +# PR_NUMBER defaults to the current branch's PR. +# +# Comments `/review` on the PR (works on drafts), waits for the spawned +# "PR Review Commands" workflow run(s) to complete, then prints one verdict +# line per reviewer and saves each full review comment to a file. A round +# takes 10-30 minutes: run this in the background and act on its output when +# it exits, per the pr skill ("Review rounds"). +set -euo pipefail + +REPO=${REPO:-$(gh repo view --json nameWithOwner --jq .nameWithOwner)} +PR=${1:-$(gh pr view --json number --jq .number)} + +# Timestamp of the trigger comment, straight from GitHub, so local clock skew +# can't make the run/comment filters below miss part of the round. +TRIGGER_TIME=$(gh api "repos/$REPO/issues/$PR/comments" -f body='/review' --jq .created_at) +echo "Review round triggered on $REPO#$PR at $TRIGGER_TIME" + +# Retry wrapper for one-off gh/API hiccups: a 45-minute wait must not die on +# a single transient failure. +retry() { + local attempt + for attempt in 1 2 3; do + if "$@"; then return 0; fi + sleep 10 + done + return 1 +} + +# The /review comment spawns one "PR Review Commands" run holding the +# claude/codex/pi jobs. Runs aren't linked to a PR, so wait on every run of +# that workflow created after the trigger: a concurrent round on another PR +# can only delay the answer, never truncate it. Every issue comment on any PR +# spawns a fast-completing parse run of the same workflow, so the round's own +# run may briefly lag the listing while unrelated runs already show completed: +# require the all-completed state to hold past a floor and across two +# consecutive polls before trusting it. +DEADLINE=$(( $(date +%s) + 45 * 60 )) +NO_RUN_DEADLINE=$(( $(date +%s) + 5 * 60 )) +MIN_WAIT_UNTIL=$(( $(date +%s) + 3 * 60 )) +STABLE=0 +FAILURES=0 +while :; do + if RUNS=$(gh run list --repo "$REPO" --workflow=pr-review-commands.yml \ + --created ">=$TRIGGER_TIME" --limit 100 --json status); then + FAILURES=0 + else + FAILURES=$(( FAILURES + 1 )) + if [ "$FAILURES" -ge 5 ]; then + echo "ERROR: listing workflow runs failed $FAILURES times in a row; aborting the wait." >&2 + exit 1 + fi + echo "WARNING: listing workflow runs failed (attempt $FAILURES/5); retrying in 60s." >&2 + sleep 60 + continue + fi + TOTAL=$(jq length <<<"$RUNS") + PENDING=$(jq '[.[] | select(.status != "completed")] | length' <<<"$RUNS") + NOW=$(date +%s) + if [ "$TOTAL" -gt 0 ] && [ "$PENDING" -eq 0 ] && [ "$NOW" -gt "$MIN_WAIT_UNTIL" ]; then + STABLE=$(( STABLE + 1 )) + if [ "$STABLE" -ge 2 ]; then + break + fi + else + STABLE=0 + fi + if [ "$TOTAL" -eq 0 ] && [ "$NOW" -gt "$NO_RUN_DEADLINE" ]; then + echo "ERROR: no 'PR Review Commands' run appeared within 5 minutes of the /review comment; check that the comment author has write access and the workflow is enabled." >&2 + exit 1 + fi + if [ "$NOW" -gt "$DEADLINE" ]; then + echo "WARNING: review round still pending after 45 minutes; reporting whatever has been posted so far." >&2 + break + fi + sleep 60 +done + +# Head SHA at trigger time. `/review` is idempotent per head: it skips an agent a +# running/successful review already covers, re-runs a cancelled/failed one in place on a +# separate head-tied run, and launches fresh only when nothing covers the head. Verdict +# reading below therefore keys off the head, not just the trigger timestamp. +HEAD_SHA=$(retry gh api "repos/$REPO/pulls/$PR" --jq .head.sha) +echo "Reviewing head $HEAD_SHA" + +# Newest non-skipped run of tied to the head ("status conclusion"), or empty +# when none exists. A re-run-in-place or an already-covering review resolves on such a +# head-tied run — separate from the pr-review-commands run waited on above (a fresh +# launch instead runs inside it, and posts after the trigger). A `skipped` run is the +# draft/fork gate and produced no review, so it is ignored. +head_run_state() { + gh run list --repo "$REPO" --workflow "$1" --commit "$HEAD_SHA" --limit 20 \ + --json databaseId,status,conclusion \ + --jq '[.[] | select(.conclusion != "skipped")] | sort_by(.databaseId) | last | if . then "\(.status) \(.conclusion // "-")" else empty end' 2>/dev/null || true +} + +# A re-run-in-place review lands on a head-tied run that finishes after the fast +# pr-review-commands run, so let those settle before reading verdicts. +for wf in codex-pr-review.yml pi-pr-review.yml pr-ready-review.yml; do + while :; do + case "$(head_run_state "$wf")" in + ""|"completed "*) break ;; + *) if [ "$(date +%s)" -gt "$DEADLINE" ]; then break; fi; sleep 30 ;; + esac + done +done + +OUT_DIR=$(mktemp -d -t review-round-XXXXXX) +COMMENTS_RAW=$(retry gh api "repos/$REPO/issues/$PR/comments?per_page=100" --paginate) +# Two views: comments from THIS round (after the trigger) and the full history. A fresh +# launch posts after the trigger; an idempotent skip leaves the covering verdict in the +# earlier run's comment, so fall back to history when that agent's head run is green. +jq -s --arg t "$TRIGGER_TIME" '[.[][] | select(.created_at > $t)]' \ + <<<"$COMMENTS_RAW" > "$OUT_DIR/comments.json" +jq -s '[.[][]]' <<<"$COMMENTS_RAW" > "$OUT_DIR/comments-all.json" +# cubic posts through the PR reviews API, not issue comments. +REVIEWS_RAW=$(retry gh api "repos/$REPO/pulls/$PR/reviews?per_page=100" --paginate) +jq -s --arg t "$TRIGGER_TIME" '[.[][] | select((.submitted_at // "") > $t)]' \ + <<<"$REVIEWS_RAW" > "$OUT_DIR/pr-reviews.json" + +VERDICT_RE='(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)' + +body_by_header() { # + jq -r --arg h "$2" '[.[] | select(.body // "" | contains($h))] | last | .body // empty' "$1" +} +body_by_login() { # + jq -r --arg l "$2" '[.[] | select(.user.login == $l)] | last | .body // empty' "$1" +} +head_ok() { [ "$(head_run_state "$1")" = "completed success" ]; } +# Latest verdict for a reviewer: prefer this round's comment; if none and the reviewer's +# head run succeeded (an idempotent /review skipped re-reviewing an already-green head), +# fall back to the covering comment from the full history. +verdict_body() { # + local body + body=$("body_by_$1" "$OUT_DIR/comments.json" "$2") + if [ -z "$body" ] && head_ok "$3"; then + body=$("body_by_$1" "$OUT_DIR/comments-all.json" "$2") + fi + printf '%s' "$body" +} +report() { # + local name=$1 body=$2 verdict + if [ -z "$body" ]; then + echo "$name: (no review posted for this head)" + return + fi + printf '%s\n' "$body" > "$OUT_DIR/$name.md" + verdict=$(printf '%s\n' "$body" | grep -m1 -oE "${VERDICT_RE}.*" | sed 's/\*\*//g' || true) + echo "$name: ${verdict:-(review posted but no verdict line; read $OUT_DIR/$name.md)}" +} + +echo +echo "=== Review round verdicts for $REPO#$PR (head $HEAD_SHA) ===" +CODEX_BODY=$(verdict_body header '## Codex Review' codex-pr-review.yml) +report codex "$CODEX_BODY" +report claude "$(verdict_body login 'claude[bot]' pr-ready-review.yml)" +report pi "$(verdict_body header '## Pi Review' pi-pr-review.yml)" +CUBIC_BODY=$(jq -r '[.[] | select(.user.login | test("^cubic(-dev-ai)?(\\[bot\\])?$"; "i"))] | last | .body // empty' \ + "$OUT_DIR/pr-reviews.json") +if [ -z "$CUBIC_BODY" ]; then + CUBIC_BODY=$(jq -r '[.[] | select(.user.login | test("^cubic(-dev-ai)?(\\[bot\\])?$"; "i"))] | last | .body // empty' \ + "$OUT_DIR/comments.json") +fi +report cubic "$CUBIC_BODY" +echo +echo "Full round output: $OUT_DIR (comments.json, pr-reviews.json, one .md per reviewer)" +if [ -z "$CODEX_BODY" ]; then + echo "WARNING: no Codex verdict for $HEAD_SHA - its head run is not green (cancelled/failed/absent, not merely skipped-because-already-reviewed). Re-trigger with a '/codex' PR comment (re-runs the interrupted run in place, or launches one) and wait again." >&2 +fi diff --git a/.claude/hooks/guard-rm-outside-tmp.sh b/.claude/hooks/guard-rm-outside-tmp.sh new file mode 100755 index 0000000000..66d4dd27b5 --- /dev/null +++ b/.claude/hooks/guard-rm-outside-tmp.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# PreToolUse guard for `rm`: auto-allow ONLY a single, plain, single-line `rm` whose every +# operand is a whitelisted target — under /tmp, or inside a git working tree located in $HOME +# (a version-controlled project dir). Anything else makes no decision (exit 0) and falls back +# to the normal permission flow, where the `Bash(rm:*)` ask rule prompts (classifier as a +# backstop). +# +# The git-tree allowance trades on "this is a project under version control" being lower-stakes +# than a delete elsewhere — NOT on full recoverability: committed content is restorable via git, +# but untracked / .gitignore'd / uncommitted content, and an independent nested repo's history +# under a recursively-deleted parent, are NOT. Accepted as a deliberate convenience tradeoff. +# +# Deny-by-default: every token must consist only of a safe character set (alphanumerics, +# `. _ / -` and glob chars `* ? [ ]`). That set contains none of the characters bash uses for +# quoting, expansion, or command separation ($ ` ~ { } ( ) ' " \ ; & | < >), so those forms +# fail by construction rather than needing to be enumerated. `realpath -m` then resolves `..` +# and existing symlinks (so a symlink out of the allowed roots is caught), and a wildcard in a +# non-final path segment is refused because it can expand through a symlink realpath can't see. +# +# The git-repo allowance covers targets inside a git working tree under $HOME, and the tree's +# own root folder only when it is a linked worktree (`.git` is a pointer file, so history in +# the main repo survives); a primary checkout's root (`.git` is a history dir) and any `.git` +# path are never auto-allowed. Globs auto-allow only under /tmp — elsewhere their expansion +# could reach `.git` or a dotfile the literal checks never see. Relative operands resolve +# against the command's cwd (from the hook input). A PreToolUse `allow` overrides the ask rule. +# +# Assumes GNU `realpath` (-m) and `jq`, both present in this repo's Linux dev env. +set -uo pipefail + +input=$(cat) +command -v jq >/dev/null 2>&1 || exit 0 +cmd=$(printf '%s' "$input" | jq -r '.tool_input.command // empty' 2>/dev/null) +[ -z "$cmd" ] && exit 0 +cwd=$(printf '%s' "$input" | jq -r '.cwd // empty' 2>/dev/null) + +# A newline separates commands, and the tokenizer below only reads the first line — defer. +case "$cmd" in *$'\n'*) exit 0 ;; esac + +read -r -a toks <<< "$cmd" +# Bare leading `rm` only; wrappers (`timeout rm`), env prefixes, and `/bin/rm` defer. +[ "${toks[0]:-}" = "rm" ] || exit 0 + +# 0 (allow) iff the canonical path is an auto-allowable rm target: under /tmp, or strictly +# inside a git working tree located under $HOME. The walk stops at $HOME, so a dotfiles repo at +# ~ can't make all of $HOME deletable, and top-level ~ files stay protected. +allowed_target() { + local canon="$1" d root="" + case "$canon" in /tmp/?*) return 0 ;; esac + [ -n "${HOME:-}" ] || return 1 + case "$canon" in "$HOME"/?*) ;; *) return 1 ;; esac + case "$canon" in *"/.git" | *"/.git/"*) return 1 ;; esac # protect history, not recoverable + d="$canon" + while [ "$d" != "/" ] && [ "$d" != "$HOME" ]; do + [ -e "$d/.git" ] && { root="$d"; break; } + d=$(dirname "$d") + done + [ -n "$root" ] || return 1 # not inside a git working tree under $HOME + if [ "$canon" = "$root" ]; then + # Deleting the repo root folder itself: allow only for a linked worktree, whose `.git` is + # a file/pointer so the history lives in the main repo and survives. A primary checkout's + # `.git` is a directory holding the history, so deleting it is unrecoverable — defer. + [ -f "$root/.git" ] && return 0 + return 1 + fi + return 0 +} + +had_operand=0 +end_opts=0 +i=1 +while [ "$i" -lt "${#toks[@]}" ]; do + t="${toks[$i]}" + i=$((i + 1)) + # Whitelist every token (flags included, so an operator hidden in a flag like `-rf;rm` + # can't slip past): any character outside the safe set makes it unsafe to reason about. + [ -n "$(printf '%s' "$t" | tr -d 'A-Za-z0-9._/*?[]-')" ] && exit 0 + # A glob in an option-looking token (`-[-]`) can expand to `--` and turn a later `-name` + # into an operand — never a real option, so defer. + case "$t" in -*[*?[]*) exit 0 ;; esac + if [ "$end_opts" = 0 ]; then + [ "$t" = "--" ] && { end_opts=1; continue; } + # Skip real options only before the first operand. A bare `-` is a filename, and under + # POSIXLY_CORRECT GNU rm stops option parsing at the first operand, so a later `-name` + # is a filename too — validate it rather than skipping it. + if [ "$had_operand" = 0 ]; then + case "$t" in -?*) continue ;; esac + fi + fi + had_operand=1 + # No wildcard in a non-final path segment (`a/*/b`): it can expand through a symlink + # realpath can't see. A slashless glob (`*.rs`) is a final-segment match — fine. + case "$t" in */*) case "${t%/*}" in *[*?[]*) exit 0 ;; esac ;; esac + case "$t" in + /*) canon=$(realpath -m -- "$t" 2>/dev/null) ;; + *) canon=$(realpath -m -- "${cwd:-$PWD}/$t" 2>/dev/null) ;; + esac + [ -n "$canon" ] || exit 0 + # A glob may auto-allow only under /tmp, where everything is deletable. Elsewhere its + # expansion could match `.git`, a dotfile like `.*`, or a nested checkout root that the + # literal-path checks never see — so require literal operands in git repos. + case "$t" in *[*?[]*) case "$canon" in /tmp/?*) ;; *) exit 0 ;; esac ;; esac + allowed_target "$canon" || exit 0 +done + +[ "$had_operand" = 1 ] || exit 0 +jq -nc '{hookSpecificOutput:{hookEventName:"PreToolUse",permissionDecision:"allow",permissionDecisionReason:"rm operands are under /tmp or inside a git checkout in $HOME"}}' diff --git a/.claude/settings.json b/.claude/settings.json index ca8d9a898d..eeb3683842 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -48,9 +48,6 @@ "Read(/tmp/**)", "Write(/tmp/**)", "Edit(/tmp/**)", - "Bash(rm:/tmp/*)", - "Bash(rm:/tmp/**)", - "Bash(rmdir:/tmp/*)", "Bash(mkdir:/tmp/*)", "Bash(mkdir:/tmp/**)", "Bash(cp:/tmp/*)", @@ -62,7 +59,12 @@ "Bash(chmod:/tmp/*)", "Bash(chmod:/tmp/**)", "Bash(tar * /tmp/*)", - "Bash(unzip * /tmp/*)" + "Bash(unzip * /tmp/*)", + "mcp__claude_ai_Gmail__search_threads", + "mcp__claude_ai_Gmail__get_thread", + "mcp__claude_ai_Gmail__get_message", + "mcp__claude_ai_Gmail__list_labels", + "mcp__claude_ai_Gmail__list_drafts" ], "deny": [ "Read(.env)", @@ -92,7 +94,15 @@ "Bash(shred:*)", "Bash(unlink:*)", "mcp__claude_ai_Stripe", - "mcp__claude_ai_Gmail", + "mcp__claude_ai_Gmail__create_draft", + "mcp__claude_ai_Gmail__update_draft", + "mcp__claude_ai_Gmail__create_label", + "mcp__claude_ai_Gmail__label_message", + "mcp__claude_ai_Gmail__label_thread", + "mcp__claude_ai_Gmail__unlabel_message", + "mcp__claude_ai_Gmail__unlabel_thread", + "mcp__claude_ai_Gmail__apply_sensitive_message_label", + "mcp__claude_ai_Gmail__apply_sensitive_thread_label", "mcp__claude_ai_Google_Calendar", "mcp__claude_ai_Google_Drive", "mcp__claude_ai_Slack", @@ -109,6 +119,11 @@ "type": "command", "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/guard-main-branch.sh", "timeout": 5 + }, + { + "type": "command", + "command": "\"$CLAUDE_PROJECT_DIR\"/.claude/hooks/guard-rm-outside-tmp.sh", + "timeout": 5 } ] } diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index b952a85e7f..5042ed9bfe 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -51,6 +51,11 @@ jobs: with: cache-workspaces: backend toolchain: 1.97.0 + # This action defaults RUSTFLAGS to "-D warnings"; unset it so the test + # run is not failed by cross-platform dead-code (cfg(unix)-only helpers + # are unused on Windows). Warning hygiene is enforced on the Linux CI + # and the build_windows_worker_ release build, not this test job. + rustflags: "" - uses: actions/setup-dotnet@v4 with: @@ -203,9 +208,15 @@ jobs: WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + # Windows ships a worker-only binary, so test the crates a worker runs + # (windmill-worker/-common/-queue) via -p, not `--all`: this skips the + # disk-heavy windmill-api test binaries (LNK1180) and the server-only + # windmill-trigger-* crates (amqp does not build on Windows). Linux CI runs the rest. run: > cargo test --no-fail-fast - --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline - --all + -p windmill-worker + -p windmill-common + -p windmill-queue + --features private,enterprise,deno_core,duckdb,python,rust,csharp,php,quickjs,parquet,mcp,scoped_cache,windmill-git-sync/private,windmill-object-store/private,windmill-object-store/enterprise -- --nocapture --test-threads=10 diff --git a/.github/workflows/build-caddy-l4-image.yml b/.github/workflows/build-caddy-l4-image.yml index e4cfdf8112..a26f1df2cc 100644 --- a/.github/workflows/build-caddy-l4-image.yml +++ b/.github/workflows/build-caddy-l4-image.yml @@ -10,6 +10,14 @@ on: - main paths: - docker/DockerfileCaddyL4 + - docker/entrypoint-caddy.sh + - docker/caddy-compat-normalize.awk + - docker/caddy-l4.version + - docker/test-caddy-compat.sh + - Caddyfile + # The version check below reads the pin out of docker-compose.yml, so a + # compose-only bump has to trigger this workflow or the check never runs. + - docker-compose.yml - .github/workflows/build-caddy-l4-image.yml permissions: write-all @@ -20,6 +28,35 @@ jobs: steps: - uses: actions/checkout@v4 - uses: depot/setup-action@v1 + + # docker-compose.yml pins an exact tag, and the Caddyfile it must agree + # with lives in the same checkout. Fail the build rather than publish a + # version nothing references, which is how :latest drifted from the + # Caddyfile in the first place. + - name: Resolve and check image version + id: version + run: | + set -euo pipefail + version="$(tr -d '[:space:]' < docker/caddy-l4.version)" + pinned="$(grep -oE 'caddy-l4:[^[:space:]"]+' docker-compose.yml | head -1 | cut -d: -f2-)" + caddy="$(grep -m1 -oE '^FROM caddy:[0-9]+\.[0-9]+\.[0-9]+' docker/DockerfileCaddyL4 | cut -d: -f2)" + if [ "$version" != "$pinned" ]; then + echo "docker/caddy-l4.version is '$version' but docker-compose.yml pins '$pinned'" >&2 + echo "Bump both together." >&2 + exit 1 + fi + # Otherwise a caddy bump that forgets the version file publishes a tag + # that names the wrong caddy. + case "$version" in + "$caddy"-*) ;; + *) + echo "docker/caddy-l4.version is '$version' but the Dockerfile pins caddy '$caddy'" >&2 + echo "The version must be -." >&2 + exit 1 + ;; + esac + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: Docker meta id: meta-ee-public uses: docker/metadata-action@v5 @@ -29,8 +66,22 @@ jobs: tags: | type=sha type=ref,event=branch + # Not gated on the default branch: docker-compose.yml pins this tag, + # so it has to be publishable from a branch (workflow_dispatch) + # before the pin merges, or main would reference a tag that does not + # exist yet. The version is immutable, so republishing from main is + # a no-op. Only branch pushes to main and manual dispatch run this + # workflow, so a branch cannot claim the tag by accident. + type=raw,value=${{ steps.version.outputs.version }} type=raw,value=latest,enable={{is_default_branch}} + # The shim rewrites config a self-hoster never sees, so a silent + # regression here strands them on a restart loop or a dead :80. + - name: Test the legacy-Caddyfile compatibility shim + run: | + docker build -f docker/DockerfileCaddyL4 -t caddy-l4:ci ./docker + docker/test-caddy-compat.sh caddy-l4:ci + - name: Login to registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index e36473f2aa..dcb223b534 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -63,6 +63,7 @@ jobs: push: true build-args: | features=ee_rhel + WM_BUILD_VERSION=${{ github.sha }} secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build-publish-rh8-image.yml b/.github/workflows/build-publish-rh8-image.yml index b7a7196077..7c820eb290 100644 --- a/.github/workflows/build-publish-rh8-image.yml +++ b/.github/workflows/build-publish-rh8-image.yml @@ -65,6 +65,7 @@ jobs: push: true build-args: | features=ee_rhel + WM_BUILD_VERSION=${{ github.sha }} secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -82,6 +83,7 @@ jobs: push: true build-args: | features=ee_rhel + WM_BUILD_VERSION=${{ github.sha }} secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 2d573c6959..84f4366235 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -45,8 +45,12 @@ jobs: env: RUSTFLAGS: "-D warnings" run: | - mkdir frontend/build && cd backend + cd backend + # Stub the openapi specs to empty: they are compiled in via an ungated + # include_str! but a worker binary never serves them, so this avoids + # embedding ~2.5MB of spec. New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force cargo check --features=ee_windows - name: Cargo build dynamic libraries windows diff --git a/.github/workflows/claude-plan.yml b/.github/workflows/claude-plan.yml index ef25554f83..bc81801417 100644 --- a/.github/workflows/claude-plan.yml +++ b/.github/workflows/claude-plan.yml @@ -49,7 +49,7 @@ jobs: allowed_bots: 'windmill-internal-app[bot]' trigger_phrase: '/plan' claude_args: | - --model claude-opus-4-8 + --model claude-opus-5 --system-prompt "# Claude Planning Mode You are operating in PLANNING MODE ONLY. Your role is to create detailed, structured plans without making any code changes. diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 115115dac6..5e9aab00d3 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -93,4 +93,4 @@ jobs: } claude_args: | --allowedTools "Bash,WebFetch,WebSearch" - --model claude-opus-4-8 + --model claude-opus-5 diff --git a/.github/workflows/codex-pr-review.yml b/.github/workflows/codex-pr-review.yml index 68bea1d715..26b2d9aae8 100644 --- a/.github/workflows/codex-pr-review.yml +++ b/.github/workflows/codex-pr-review.yml @@ -82,6 +82,7 @@ jobs: EVENT_BODY: ${{ github.event.pull_request.body }} EVENT_FORK: ${{ github.event.pull_request.head.repo.fork }} EVENT_AUTHOR: ${{ github.event.pull_request.user.login }} + EVENT_ACTION: ${{ github.event.action }} run: | if [ -n "$INPUT_PR_NUMBER" ]; then PR_JSON=$(gh pr view "$INPUT_PR_NUMBER" --repo "${{ github.repository }}" \ @@ -113,6 +114,38 @@ jobs: echo "skip=true" >> "$GITHUB_OUTPUT" exit 0 fi + # An agent-driven PR flips to ready only after a clean /review round on + # a draft, marked by an author comment naming the head SHA (pr skill, + # "Review rounds"). Re-reviewing that same head on ready_for_review is + # redundant. The marker alone is author attestation, so also require + # reviewer evidence: a Codex review (posted by github-actions[bot], not + # forgeable by the author) that predates the marker and carries a + # non-blocking verdict. Comment-triggered and synchronize runs never + # skip. Keep the three copies of this check in sync (pr-ready-review / + # codex-pr-review / pi-pr-review); a shared local action would need the + # repo checked out before the check, which the fork paths here + # deliberately avoid. + if [ "$EVENT_ACTION" = "ready_for_review" ] && [ -z "$INPUT_PR_NUMBER" ]; then + # Fetch failures fail open (no skip): an API hiccup must run the + # review, never skip it or fail the job. + COMMENTS=$(gh api "repos/${{ github.repository }}/issues/$PR_NUMBER/comments?per_page=100" --paginate | jq -s '[.[][]]') || COMMENTS='[]' + MARKER_TIME=$(jq -r --arg author "$PR_AUTHOR" --arg marker "✅ Review round clean @ $HEAD_SHA" \ + '[.[] | select(.user.login == $author) | select(.body | contains($marker)) | .created_at] | min // empty' <<<"$COMMENTS") + CODEX_VERDICT='' + if [ -n "$MARKER_TIME" ]; then + # Only Codex evidence that predates the marker counts: the ready- + # triggered Codex run itself posts after the flip and must not + # vouch for a sibling reviewer's skip. + CODEX_VERDICT=$(jq -r --arg mt "$MARKER_TIME" \ + '[.[] | select(.user.login == "github-actions[bot]") | select(.body | contains("## Codex Review")) | select(.created_at < $mt)] | last | .body // ""' <<<"$COMMENTS" \ + | grep -m1 -oE '(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)' || true) + fi + if [ -n "$MARKER_TIME" ] && [ -n "$CODEX_VERDICT" ] && [ "$CODEX_VERDICT" != "Should address issues before merging" ]; then + echo "Clean review round marker found for $HEAD_SHA with pre-marker non-blocking Codex verdict; skipping redundant review." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi # PR title/body are attacker-controlled free text. Use an unguessable # per-run delimiter so a fork can't embed a fixed heredoc terminator to # inject extra outputs — e.g. is_fork=false (last-write-wins), which diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 4a7fc2a874..fa5f4ba3fe 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -68,6 +68,7 @@ jobs: push: true build-args: | features=ce_rpi + WM_BUILD_VERSION=${{ github.sha }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index bdc53a6f22..433f3a86b3 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -93,6 +93,7 @@ jobs: push: true build-args: | features=ce + WM_BUILD_VERSION=${{ github.sha }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -155,6 +156,7 @@ jobs: push: true build-args: | features=ee + WM_BUILD_VERSION=${{ github.sha }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -254,6 +256,7 @@ jobs: target: debuginfo build-args: | features=ee + WM_BUILD_VERSION=${{ github.sha }} outputs: type=local,dest=./debuginfo - name: Rename debug file with corresponding architecture diff --git a/.github/workflows/git-sync-test.yml b/.github/workflows/git-sync-test.yml index 3ff7d8b85c..4b99c62eae 100644 --- a/.github/workflows/git-sync-test.yml +++ b/.github/workflows/git-sync-test.yml @@ -9,6 +9,10 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "backend/windmill-worker/src/result_processor.rs" + - "backend/windmill-api-workspaces/**" + - "cli/src/commands/sync/**" + - "cli/src/utils/git.ts" - "integration_tests/test/git_sync_test.py" - ".github/workflows/git-sync-test.yml" pull_request: @@ -18,6 +22,10 @@ on: - "backend/windmill-api-integration-tests/tests/git_sync*" - "backend/ee-repo-ref.txt" - "backend/windmill-common/src/workspaces.rs" + - "backend/windmill-worker/src/result_processor.rs" + - "backend/windmill-api-workspaces/**" + - "cli/src/commands/sync/**" + - "cli/src/utils/git.ts" - "integration_tests/test/git_sync_test.py" - ".github/workflows/git-sync-test.yml" @@ -50,8 +58,8 @@ jobs: echo "Changed files:" echo "$CHANGED_FILES" - # Direct git sync file changes — always relevant - if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then + # Direct git sync file changes — always relevant. + if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-worker/src/result_processor\.rs|backend/windmill-api-workspaces/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|cli/src/commands/sync/|cli/src/utils/git\.ts|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then echo "should_run=true" >> "$GITHUB_OUTPUT" echo "Relevant: direct git sync file changes" exit 0 @@ -180,6 +188,9 @@ jobs: DENO_PATH: deno BUN_PATH: bun NODE_BIN_PATH: node + # The auto-pull poller's SSRF guard rejects localhost git remotes; + # the tests' Gitea runs on localhost. + ALLOW_LOCAL_GIT_REMOTES: "true" run: | ./target/debug/windmill & echo "Waiting for Windmill to be ready..." diff --git a/.github/workflows/pi-pr-review.yml b/.github/workflows/pi-pr-review.yml index 03c9599480..2f143663c8 100644 --- a/.github/workflows/pi-pr-review.yml +++ b/.github/workflows/pi-pr-review.yml @@ -75,6 +75,7 @@ jobs: EVENT_BODY: ${{ github.event.pull_request.body }} EVENT_FORK: ${{ github.event.pull_request.head.repo.fork }} EVENT_AUTHOR: ${{ github.event.pull_request.user.login }} + EVENT_ACTION: ${{ github.event.action }} run: | if [ -n "$INPUT_PR_NUMBER" ]; then PR_JSON=$(gh pr view "$INPUT_PR_NUMBER" --repo "${{ github.repository }}" \ @@ -106,6 +107,38 @@ jobs: echo "skip=true" >> "$GITHUB_OUTPUT" exit 0 fi + # An agent-driven PR flips to ready only after a clean /review round on + # a draft, marked by an author comment naming the head SHA (pr skill, + # "Review rounds"). Re-reviewing that same head on ready_for_review is + # redundant. The marker alone is author attestation, so also require + # reviewer evidence: a Codex review (posted by github-actions[bot], not + # forgeable by the author) that predates the marker and carries a + # non-blocking verdict. Comment-triggered and synchronize runs never + # skip. Keep the three copies of this check in sync (pr-ready-review / + # codex-pr-review / pi-pr-review); a shared local action would need the + # repo checked out before the check, which the fork paths here + # deliberately avoid. + if [ "$EVENT_ACTION" = "ready_for_review" ] && [ -z "$INPUT_PR_NUMBER" ]; then + # Fetch failures fail open (no skip): an API hiccup must run the + # review, never skip it or fail the job. + COMMENTS=$(gh api "repos/${{ github.repository }}/issues/$PR_NUMBER/comments?per_page=100" --paginate | jq -s '[.[][]]') || COMMENTS='[]' + MARKER_TIME=$(jq -r --arg author "$PR_AUTHOR" --arg marker "✅ Review round clean @ $HEAD_SHA" \ + '[.[] | select(.user.login == $author) | select(.body | contains($marker)) | .created_at] | min // empty' <<<"$COMMENTS") + CODEX_VERDICT='' + if [ -n "$MARKER_TIME" ]; then + # Only Codex evidence that predates the marker counts: the ready- + # triggered Codex run itself posts after the flip and must not + # vouch for a sibling reviewer's skip. + CODEX_VERDICT=$(jq -r --arg mt "$MARKER_TIME" \ + '[.[] | select(.user.login == "github-actions[bot]") | select(.body | contains("## Codex Review")) | select(.created_at < $mt)] | last | .body // ""' <<<"$COMMENTS" \ + | grep -m1 -oE '(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)' || true) + fi + if [ -n "$MARKER_TIME" ] && [ -n "$CODEX_VERDICT" ] && [ "$CODEX_VERDICT" != "Should address issues before merging" ]; then + echo "Clean review round marker found for $HEAD_SHA with pre-marker non-blocking Codex verdict; skipping redundant review." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + fi # PR title/body are attacker-controlled free text. Use an unguessable # per-run delimiter so a fork can't embed a fixed heredoc terminator to # inject extra outputs — e.g. is_fork=false (last-write-wins), which diff --git a/.github/workflows/pr-ready-review.yml b/.github/workflows/pr-ready-review.yml index eb634977d1..d2acde0dbe 100644 --- a/.github/workflows/pr-ready-review.yml +++ b/.github/workflows/pr-ready-review.yml @@ -48,12 +48,55 @@ jobs: pull-requests: read id-token: write steps: + # An agent-driven PR flips to ready only after a clean /review round on a + # draft, marked by an author comment naming the head SHA (pr skill, "Review + # rounds"). Re-reviewing that same head on ready_for_review is redundant. + # The marker alone is author attestation, so also require reviewer evidence: + # a Codex review (posted by github-actions[bot], not forgeable by the author) + # that predates the marker and carries a non-blocking verdict. Comment- + # triggered (workflow_call) and opened runs never skip. Keep the three + # copies of this check in sync (pr-ready-review / codex-pr-review / + # pi-pr-review); a shared local action would need the repo checked out + # before the check, which the codex/pi fork paths deliberately avoid. + - name: Check clean-round marker + id: marker + if: github.event_name == 'pull_request' && github.event.action == 'ready_for_review' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + # Fetch failures fail open (skip=false): an API hiccup must run the + # review, never skip it or fail the job. + COMMENTS=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments?per_page=100" --paginate | jq -s '[.[][]]') || COMMENTS='[]' + MARKER_TIME=$(jq -r --arg author "$PR_AUTHOR" --arg marker "✅ Review round clean @ $HEAD_SHA" \ + '[.[] | select(.user.login == $author) | select(.body | contains($marker)) | .created_at] | min // empty' <<<"$COMMENTS") + CODEX_VERDICT='' + if [ -n "$MARKER_TIME" ]; then + # Only Codex evidence that predates the marker counts: the ready- + # triggered Codex run itself posts after the flip and must not vouch + # for a sibling reviewer's skip. + CODEX_VERDICT=$(jq -r --arg mt "$MARKER_TIME" \ + '[.[] | select(.user.login == "github-actions[bot]") | select(.body | contains("## Codex Review")) | select(.created_at < $mt)] | last | .body // ""' <<<"$COMMENTS" \ + | grep -m1 -oE '(Good to merge|Mergeable, but should ideally address nits|Should address issues before merging)' || true) + fi + if [ -n "$MARKER_TIME" ] && [ -n "$CODEX_VERDICT" ] && [ "$CODEX_VERDICT" != "Should address issues before merging" ]; then + echo "Clean review round marker found for $HEAD_SHA with pre-marker non-blocking Codex verdict; skipping redundant review." + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + - name: Checkout repository + if: steps.marker.outputs.skip != 'true' uses: actions/checkout@v5 with: fetch-depth: 1 - name: Check EE access + if: steps.marker.outputs.skip != 'true' id: ee env: EE_TOKEN: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} @@ -80,6 +123,7 @@ jobs: run: ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private - name: Resolve PR number + if: steps.marker.outputs.skip != 'true' id: resolve env: GH_TOKEN: ${{ github.token }} @@ -99,6 +143,7 @@ jobs: echo "pr_author=$PR_AUTHOR" >> "$GITHUB_OUTPUT" - name: Fetch prior PR discussion + if: steps.marker.outputs.skip != 'true' id: prior env: GH_TOKEN: ${{ github.token }} @@ -117,6 +162,7 @@ jobs: ' prior-comments.json > prior-comments.md - name: Read review prompt + if: steps.marker.outputs.skip != 'true' id: review-prompt env: EXTRA_PROMPT: ${{ inputs.extra_prompt }} @@ -140,6 +186,7 @@ jobs: } >> "$GITHUB_ENV" - name: Automatic PR Review + if: steps.marker.outputs.skip != 'true' uses: anthropics/claude-code-action@v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} @@ -152,4 +199,4 @@ jobs: ${{ env.REVIEW_PROMPT }} claude_args: | --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*)" - --model claude-opus-4-8 + --model claude-opus-5 diff --git a/.github/workflows/pr-review-commands.yml b/.github/workflows/pr-review-commands.yml index ef93274d7e..3c04e4e5f2 100644 --- a/.github/workflows/pr-review-commands.yml +++ b/.github/workflows/pr-review-commands.yml @@ -75,14 +75,153 @@ jobs: "/repos/$REPO/issues/comments/$COMMENT_ID/reactions" \ -f content=eyes >/dev/null - claude: + # Decide, per agent, whether to launch a fresh run, re-run in place, or skip. A push + # already auto-triggers codex/pi (and claude on open) against the PR head. Relaunching + # via this issue_comment path both cancels those in-flight auto runs (shared concurrency + # group) AND lands the new run's status on main — issue_comment runs never attach a + # check to the PR head — leaving the PR showing only a cancelled review. So for every + # command, launch an agent only when nothing covers the head commit; if the head's run + # was cancelled/failed, re-run it in place (a re-run keeps the original pull_request + # event, so its checks re-attach to the PR head); skip when a running or successful run + # already covers it. `/review` applies this to all three agents; `/codex`, `/pi`, + # `/claude` apply the same decision to just their own agent. + plan: needs: [parse, check-access] + if: | + needs.parse.outputs.command != '' && + ( + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || + needs.check-access.outputs.authorized == 'true' + ) + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + pull-requests: read + statuses: write + outputs: + head_sha: ${{ steps.plan.outputs.head_sha }} + launch_codex: ${{ steps.plan.outputs.launch_codex }} + launch_pi: ${{ steps.plan.outputs.launch_pi }} + launch_claude: ${{ steps.plan.outputs.launch_claude }} + steps: + - name: Decide per-agent launch vs re-run for the head commit + id: plan + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMAND: ${{ needs.parse.outputs.command }} + run: | + set -euo pipefail + + HEAD_SHA=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json headRefOid --jq '.headRefOid') + echo "PR #$PR_NUMBER head: $HEAD_SHA" + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + RUN_URL="$GITHUB_SERVER_URL/$REPO/actions/runs/$GITHUB_RUN_ID" + + # A fresh launch runs from this issue_comment workflow (associated with main), + # so it never appears in the PR-head run query below and its own check lands on + # main, not the head. To keep fresh launches idempotent per head, mark the head + # SHA with a `review-launch/` commit status at launch; the `finalize` job + # resolves it to success/failure. A prior launch's status covering the head lets + # a second comment skip instead of relaunching (which would cancel the first via + # the reviewer's shared concurrency group). All status calls are best-effort — a + # GitHub API hiccup must degrade to a relaunch, never abort the decision. + mark_launch() { + agent="$1" + gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \ + -f state=pending -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \ + -f "description=Review launched via /$COMMAND" >/dev/null 2>&1 || true + } + + # Returns "covered" if a prior fresh launch (this or an earlier comment run) + # already covers the head: a success status, or a pending status whose launching + # run is still alive. A pending whose run has completed is stale (that run + # crashed before finalize) and does not count. + launch_coverage() { + agent="$1" + st_json=$(gh api "repos/$REPO/commits/$HEAD_SHA/statuses" \ + --jq "[.[] | select(.context == \"review-launch/$agent\")] | first // empty" 2>/dev/null || true) + [ -n "$st_json" ] || return 0 + state=$(jq -r '.state // empty' <<<"$st_json" 2>/dev/null || true) + [ "$state" = success ] && { echo covered; return 0; } + [ "$state" = pending ] || return 0 + target=$(jq -r '.target_url // empty' <<<"$st_json" 2>/dev/null || true) + run_id=$(printf '%s' "$target" | grep -oE '[0-9]+$' || true) + if [ -n "$run_id" ]; then + run_state=$(gh run view "$run_id" --repo "$REPO" --json status --jq '.status' 2>/dev/null || true) + [ "$run_state" = completed ] && return 0 # stale pending -> not covered + fi + echo covered + } + + decide() { + wf="$1"; key="$2"; agent="$3" + if [ "$(launch_coverage "$agent")" = covered ]; then + echo "$key: a prior launch already covers $HEAD_SHA (review-launch/$agent) -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + # `--commit` matches runs whose head SHA is the PR head. Auto reviews run on + # `pull_request` against that SHA; `/review` (issue_comment) runs execute on + # main, so they never match and are not counted as covering the head commit. + runs=$(gh run list --repo "$REPO" --workflow "$wf" --commit "$HEAD_SHA" --limit 40 \ + --json databaseId,status,conclusion) + # Healthy = still running, or completed successfully: a review already + # covers this commit, so skip. + healthy=$(jq -r '[.[] | select(.status != "completed" or .conclusion == "success")] | length' <<<"$runs") + if [ "$healthy" -gt 0 ]; then + echo "$key: a running or successful review already covers $HEAD_SHA -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + # Re-run only genuinely interrupted runs (cancelled/failed/timed out) in + # place, so their checks re-attach to the PR head instead of posting on + # main. A `skipped` run produced no review and would just skip again (it is + # the draft/fork gate), so it does not count — fall through to a fresh launch. + retry_id=$(jq -r '[.[] | select(.status == "completed" and (.conclusion == "cancelled" or .conclusion == "failure" or .conclusion == "timed_out"))] | sort_by(.databaseId) | last | .databaseId // empty' <<<"$runs") + if [ -n "$retry_id" ]; then + if gh run rerun "$retry_id" --repo "$REPO" >/dev/null 2>&1; then + echo "$key: re-ran interrupted run $retry_id (re-attaches to PR head)" + echo "$key=false" >> "$GITHUB_OUTPUT" + return + fi + echo "$key: re-run of $retry_id failed -> fresh launch" + mark_launch "$agent" + echo "$key=true" >> "$GITHUB_OUTPUT" + return + fi + echo "$key: no usable review for $HEAD_SHA -> launch" + mark_launch "$agent" + echo "$key=true" >> "$GITHUB_OUTPUT" + } + + # `/review` targets all three agents; `/codex`, `/pi`, `/claude` target only + # their own. A non-targeted agent is left untouched (no launch, no re-run). + decide_if_targeted() { + wf="$1"; key="$2"; agent="$3" + if [ "$COMMAND" = review ] || [ "$COMMAND" = "$agent" ]; then + decide "$wf" "$key" "$agent" + else + echo "$key: /$COMMAND does not target $agent -> skip" + echo "$key=false" >> "$GITHUB_OUTPUT" + fi + } + + decide_if_targeted codex-pr-review.yml launch_codex codex + decide_if_targeted pi-pr-review.yml launch_pi pi + decide_if_targeted pr-ready-review.yml launch_claude claude + + claude: + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'claude') + needs.plan.outputs.launch_claude == 'true' permissions: contents: read pull-requests: read @@ -97,13 +236,13 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} codex: - needs: [parse, check-access] + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'codex') + needs.plan.outputs.launch_codex == 'true' permissions: contents: read issues: write @@ -119,13 +258,13 @@ jobs: WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} pi: - needs: [parse, check-access] + needs: [parse, check-access, plan] if: | ( contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) || needs.check-access.outputs.authorized == 'true' ) && - (needs.parse.outputs.command == 'review' || needs.parse.outputs.command == 'pi') + needs.plan.outputs.launch_pi == 'true' permissions: contents: read issues: write @@ -138,3 +277,34 @@ jobs: secrets: DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + + # Resolve the `review-launch/` head statuses that `plan` set to pending, so a + # fresh launch's outcome is visible on the PR head (not just on main) and never lingers + # as a stale pending check. Targets the exact SHA `plan` launched against, so a push + # that moved the head mid-review does not stamp a status on the new head. + finalize: + needs: [plan, claude, codex, pi] + if: always() && needs.plan.result == 'success' && needs.plan.outputs.head_sha != '' + runs-on: ubuntu-latest + permissions: + statuses: write + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ needs.plan.outputs.head_sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Finalize launch statuses on the PR head + run: | + set -uo pipefail + finalize() { + agent="$1"; launched="$2"; result="$3" + [ "$launched" = true ] || return 0 + state=$([ "$result" = success ] && echo success || echo failure) + gh api -X POST "repos/$REPO/statuses/$HEAD_SHA" \ + -f "state=$state" -f "context=review-launch/$agent" -f "target_url=$RUN_URL" \ + -f "description=Review $result" >/dev/null 2>&1 || true + } + finalize codex "${{ needs.plan.outputs.launch_codex }}" "${{ needs.codex.result }}" + finalize pi "${{ needs.plan.outputs.launch_pi }}" "${{ needs.pi.result }}" + finalize claude "${{ needs.plan.outputs.launch_claude }}" "${{ needs.claude.result }}" diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index 019de201e6..d159619fba 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -56,8 +56,12 @@ jobs: vcpkg.exe integrate install $env:VCPKGRS_DYNAMIC=1 $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" - mkdir frontend/build && cd backend + cd backend + # Stub the openapi specs to empty: they are compiled in via an ungated + # include_str! but a worker binary never serves them, so this avoids + # embedding ~2.5MB of spec. New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + New-Item -Path . -Name "windmill-api/openapi-deref.json" -ItemType "File" -Force cargo build --release --features=ee_windows - name: Rename binary with corresponding architecture run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ca0d1da5b..7fb6624a0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,282 @@ # Changelog +## [1.770.0](https://github.com/windmill-labs/windmill/compare/v1.769.0...v1.770.0) (2026-07-24) + + +### Features + +* add explore button for object storage resources ([#10306](https://github.com/windmill-labs/windmill/issues/10306)) ([28a79ce](https://github.com/windmill-labs/windmill/commit/28a79ced155e6bb3540ab76feb5415ea8f3d8f8f)) +* Add image when publishing a project ([#10310](https://github.com/windmill-labs/windmill/issues/10310)) ([48618cf](https://github.com/windmill-labs/windmill/commit/48618cff8c35a345babd12844653f83addbcd7e8)) + + +### Bug Fixes + +* do not apply workspace display name on git-sync pull ([#10308](https://github.com/windmill-labs/windmill/issues/10308)) ([992ed01](https://github.com/windmill-labs/windmill/commit/992ed0124492f5c6b9e3dcc6a309caeda8d6a7b1)) +* pass Windows system env vars to R renv install subprocess ([#10313](https://github.com/windmill-labs/windmill/issues/10313)) ([85008e4](https://github.com/windmill-labs/windmill/commit/85008e47b4a21313ec66496d3c68eb746fc8a151)) +* show scheduled singlestepflow runs in flow history sidebar ([#10312](https://github.com/windmill-labs/windmill/issues/10312)) ([f00fcb2](https://github.com/windmill-labs/windmill/commit/f00fcb2d1b1030b47c824bdefc9f0d12499bc6b6)) +* WAC wait_for_approval reads its own approval result, not the first ([#10314](https://github.com/windmill-labs/windmill/issues/10314)) ([2143d45](https://github.com/windmill-labs/windmill/commit/2143d45815af7a95182721aeb0dc937a12c5759f)) + +## [1.769.0](https://github.com/windmill-labs/windmill/compare/v1.768.0...v1.769.0) (2026-07-24) + + +### Features + +* **hub:** surface data pipelines in deploy-to-hub drawer ([#10299](https://github.com/windmill-labs/windmill/issues/10299)) ([010059a](https://github.com/windmill-labs/windmill/commit/010059a449f4d0cde0656086f1b3f03a02a1c78c)) +* **pipeline:** collapse secondary top-bar controls into an overflow menu ([#10300](https://github.com/windmill-labs/windmill/issues/10300)) ([1d25d75](https://github.com/windmill-labs/windmill/commit/1d25d7539ed7fb17c47a732d6b1cd7fc19202a04)) + + +### Bug Fixes + +* pin table actions column so it stays visible on narrow screens ([#10301](https://github.com/windmill-labs/windmill/issues/10301)) ([75acf72](https://github.com/windmill-labs/windmill/commit/75acf7207b151f0227600be555bfa5e14dfa8dd1)) +* pin validated DNS address to close SSRF DNS-rebinding TOCTOU ([#10303](https://github.com/windmill-labs/windmill/issues/10303)) ([3cf7a39](https://github.com/windmill-labs/windmill/commit/3cf7a390a37c07a818248c95defa4ecf5bf262e5)) + + +### Performance Improvements + +* optimize get_datatable_full_schema to avoid timeout on large catalogs ([#10304](https://github.com/windmill-labs/windmill/issues/10304)) ([1478d12](https://github.com/windmill-labs/windmill/commit/1478d12eb352b1b7906ccfe3ab5eaafe60ffbe4f)) + +## [1.768.0](https://github.com/windmill-labs/windmill/compare/v1.767.0...v1.768.0) (2026-07-24) + + +### Features + +* Add section to deploy projects to hub ([#9332](https://github.com/windmill-labs/windmill/issues/9332)) ([30eedf9](https://github.com/windmill-labs/windmill/commit/30eedf9ee1754cb2bbf46e82d2b3eac766bd3e6d)) +* **ai-chat:** let the session/global chat create email triggers ([#10282](https://github.com/windmill-labs/windmill/issues/10282)) ([248c875](https://github.com/windmill-labs/windmill/commit/248c8751b1fd5d6f7506a7c3fc8f500dd98af26e)) +* **ai:** enable data pipelines in AI sessions with alpha notice ([#10273](https://github.com/windmill-labs/windmill/issues/10273)) ([1f912f4](https://github.com/windmill-labs/windmill/commit/1f912f410401ec68789d43973d6e0a1f8e595f86)) +* alert on expired online license key ([#10295](https://github.com/windmill-labs/windmill/issues/10295)) ([30d8104](https://github.com/windmill-labs/windmill/commit/30d8104edcbdb7f0c63f186d22ac03000e0d4f07)) +* data-pipeline recorder, interactive player, and deploy-to-hub recording (WIN-2156) ([#10055](https://github.com/windmill-labs/windmill/issues/10055)) ([65e5041](https://github.com/windmill-labs/windmill/commit/65e504146d832b105603ceec89c6de0ad1d66e32)) +* let a workspace fall back to the instance critical alert channels ([#10292](https://github.com/windmill-labs/windmill/issues/10292)) ([717e38a](https://github.com/windmill-labs/windmill/commit/717e38a0c6b5bb2340e236a9d49645a4cebf4849)) +* **monitor:** make between-steps zombie flows hand-recoverable ([#10287](https://github.com/windmill-labs/windmill/issues/10287)) ([f02df7f](https://github.com/windmill-labs/windmill/commit/f02df7fc454b0c2a2afa9ae9848e26af0e379246)) +* surface workspace-script advanced settings in flow editor ([#10289](https://github.com/windmill-labs/windmill/issues/10289)) ([bf16e7d](https://github.com/windmill-labs/windmill/commit/bf16e7d49a7486d37cf9eb1907e80abae47a78a7)) +* **windows:** enable ruby and rlang on the windows worker ([#10279](https://github.com/windmill-labs/windmill/issues/10279)) ([0f1b864](https://github.com/windmill-labs/windmill/commit/0f1b8641f26f0eda8a8517db1a37efafdb173464)) + + +### Bug Fixes + +* **ci:** make /review idempotent per head commit, re-run cancelled reviews in place ([#10283](https://github.com/windmill-labs/windmill/issues/10283)) ([3aaceb7](https://github.com/windmill-labs/windmill/commit/3aaceb7efb3bd232da73b79fcdcf4c712caad7d3)) +* **cli:** surface shared UI changes in sync push dry-run preview ([#10278](https://github.com/windmill-labs/windmill/issues/10278)) ([14c29b7](https://github.com/windmill-labs/windmill/commit/14c29b77e90842b8c3d4395ce50cb44666cd998c)) +* **jobs:** sanitize NUL in completed job result before jsonb insert ([#10274](https://github.com/windmill-labs/windmill/issues/10274)) ([c50a2ab](https://github.com/windmill-labs/windmill/commit/c50a2abad0c222bf8f76ff10b864450120ee9ee7)) +* **monitor:** diagnose zombie-flow OOM on the transition worker, not q.worker ([#10286](https://github.com/windmill-labs/windmill/issues/10286)) ([fa36442](https://github.com/windmill-labs/windmill/commit/fa3644281f30d90aa0c4ca86520a38db5d8b91f0)) +* resolve svelte/style export conditions in raw-app CLI bundler ([#10294](https://github.com/windmill-labs/windmill/issues/10294)) ([9713e60](https://github.com/windmill-labs/windmill/commit/9713e6074d2df55db1331cf31ee1357606b1dbe9)) +* **resources:** apply resource_type changes on update (git-sync pull) — Fixes GIT-932 ([#10277](https://github.com/windmill-labs/windmill/issues/10277)) ([d7a0078](https://github.com/windmill-labs/windmill/commit/d7a0078b58f74f1ffecc03b348e8935a70f02e55)) +* surface workspace ids on duplicate names and explain fork promotion ([#10291](https://github.com/windmill-labs/windmill/issues/10291)) ([9b182aa](https://github.com/windmill-labs/windmill/commit/9b182aaf3879d4d81b6a785222dca75f085e6fb7)) +* treat concurrent_limit/timeout <= 0 as unset instead of a zero cap ([#10288](https://github.com/windmill-labs/windmill/issues/10288)) ([8eb36ce](https://github.com/windmill-labs/windmill/commit/8eb36ce008b4efe2be9a9bfebc91af68070f7a6c)) + +## [1.767.0](https://github.com/windmill-labs/windmill/compare/v1.766.2...v1.767.0) (2026-07-22) + + +### Features + +* **ai-chat:** email triggers in flow/script chat + trigger-intent eval guards (WIN-2228) ([#10267](https://github.com/windmill-labs/windmill/issues/10267)) ([e41440b](https://github.com/windmill-labs/windmill/commit/e41440b344d94ebe5746a89237c3d1be522aa59d)) +* **ai:** improve data-pipeline building in AI sessions (prompt + evals + e2e) ([#10270](https://github.com/windmill-labs/windmill/issues/10270)) ([0819641](https://github.com/windmill-labs/windmill/commit/0819641f3a89abdf114cc5e341a9f585e5d71296)) + + +### Bug Fixes + +* **ai-chat:** improve resource-type search tool description and scoring ([#10272](https://github.com/windmill-labs/windmill/issues/10272)) ([ad53673](https://github.com/windmill-labs/windmill/commit/ad53673a2855e787af9fe10dcc8998ffc7f1b960)) +* **embeddings:** retry on failed init instead of disabling for a day ([#10266](https://github.com/windmill-labs/windmill/issues/10266)) ([2318481](https://github.com/windmill-labs/windmill/commit/2318481f4f2932dc578d5cb64c4c705cf5c324a0)) +* manual resource type sync fetches from hub first, cache as fallback ([#10269](https://github.com/windmill-labs/windmill/issues/10269)) ([07d4b67](https://github.com/windmill-labs/windmill/commit/07d4b674f1dbd24c89f7d1e77094a40e528a3740)) + +## [1.766.2](https://github.com/windmill-labs/windmill/compare/v1.766.1...v1.766.2) (2026-07-22) + + +### Bug Fixes + +* **python,windows:** cross-platform cross-process wheel-install lock ([#10264](https://github.com/windmill-labs/windmill/issues/10264)) ([7e2f1af](https://github.com/windmill-labs/windmill/commit/7e2f1afffb4982099870e7ce32592cf3a447e513)) + +## [1.766.1](https://github.com/windmill-labs/windmill/compare/v1.766.0...v1.766.1) (2026-07-22) + + +### Bug Fixes + +* **jobs:** enforce self_approval_disabled on the UI resume path ([#10262](https://github.com/windmill-labs/windmill/issues/10262)) ([2d24b3a](https://github.com/windmill-labs/windmill/commit/2d24b3ac49b65469072bca929730d2c9ecd54b8f)) + +## [1.766.0](https://github.com/windmill-labs/windmill/compare/v1.765.0...v1.766.0) (2026-07-22) + + +### Features + +* **ai-sessions:** show item preview cards for tools ([#10254](https://github.com/windmill-labs/windmill/issues/10254)) ([703744f](https://github.com/windmill-labs/windmill/commit/703744fb8b76f70384cbc3341d5e1b3dbf30455e)) +* make content search a full CE feature ([#10252](https://github.com/windmill-labs/windmill/issues/10252)) ([d2c5d6f](https://github.com/windmill-labs/windmill/commit/d2c5d6f4b4ede9449407ed3831713d3e3d1d6972)) +* **sessions:** ship AI sessions as beta with legacy-chat opt-out ([#10242](https://github.com/windmill-labs/windmill/issues/10242)) ([0508cdd](https://github.com/windmill-labs/windmill/commit/0508cddf0a86b6a6a043cb31978ca68b76a95d02)) + + +### Bug Fixes + +* accept ssh/scheme-less git repo urls and $var: refs in app repo resolution ([#10246](https://github.com/windmill-labs/windmill/issues/10246)) ([b948efd](https://github.com/windmill-labs/windmill/commit/b948efd3c81aada103bd141eae03fbe626bf6ee5)) +* **copilot:** stop write_flow forcing rawscript code into nested JSON ([#10260](https://github.com/windmill-labs/windmill/issues/10260)) ([ecb1a92](https://github.com/windmill-labs/windmill/commit/ecb1a92070fb18048a6a1467270469e8462a3cbf)) +* prevent u16 underflow in suspend count causing permanent flow deadlock ([#10256](https://github.com/windmill-labs/windmill/issues/10256)) ([68b1fcc](https://github.com/windmill-labs/windmill/commit/68b1fcc5cdd3d217ebb1b9e09c0d52334faf0a63)) +* **prompts:** prefer Bun over Deno for TypeScript runtime selection ([#10253](https://github.com/windmill-labs/windmill/issues/10253)) ([380cf75](https://github.com/windmill-labs/windmill/commit/380cf752ca8fc57eb70f7c8efd57290f7902a030)) +* **tutorials:** repair broken frontend tutorials after UI redesigns ([#10255](https://github.com/windmill-labs/windmill/issues/10255)) ([5685981](https://github.com/windmill-labs/windmill/commit/5685981c9902ff37c3bd271dc5c5ede948a0787a)) + +## [1.765.0](https://github.com/windmill-labs/windmill/compare/v1.764.0...v1.765.0) (2026-07-21) + + +### Features + +* **ai:** open the Compare & Deploy page from chat with item preselection ([#10232](https://github.com/windmill-labs/windmill/issues/10232)) ([572d69e](https://github.com/windmill-labs/windmill/commit/572d69e5ae8ae12207dca175fdb5ea378d852a45)) +* **ai:** session chat nits — empty sends, command picker polish, session-state prompt ([#10233](https://github.com/windmill-labs/windmill/issues/10233)) ([9bc1f62](https://github.com/windmill-labs/windmill/commit/9bc1f6212837155c964a7db7103aabbcf14cbce2)) +* attach text files to chat messages, read on demand via file tools ([#10215](https://github.com/windmill-labs/windmill/issues/10215)) ([d6cf1ef](https://github.com/windmill-labs/windmill/commit/d6cf1ef9872cd0db0f51e4881237bac383e45f61)) +* **git-sync:** enable per-item promotion mode on dev workspaces ([#10205](https://github.com/windmill-labs/windmill/issues/10205)) ([2ce21c9](https://github.com/windmill-labs/windmill/commit/2ce21c9ef86c608d73e12a7e278d2f328e06b6fa)) +* **pipelines:** catalog declared measures and dimensions ([#10190](https://github.com/windmill-labs/windmill/issues/10190)) ([fd51d40](https://github.com/windmill-labs/windmill/commit/fd51d40f1254a206de1f38e59cc3dadce8e13aa5)) +* **triggers:** add AMQP (RabbitMQ) trigger via lapin ([#10230](https://github.com/windmill-labs/windmill/issues/10230)) ([68debab](https://github.com/windmill-labs/windmill/commit/68debab877c6dc8ee3732e0c23d467db85fd4584)) +* unified read-only `diff` chat tool (drafts, fork vs parent, search) ([#10211](https://github.com/windmill-labs/windmill/issues/10211)) ([9739d5a](https://github.com/windmill-labs/windmill/commit/9739d5a2c2a21beded8848ea9049b2816196a898)) + + +### Bug Fixes + +* **apps:** let entitled viewers read pre-existing S3 files from deployed apps ([#10245](https://github.com/windmill-labs/windmill/issues/10245)) ([4a89824](https://github.com/windmill-labs/windmill/commit/4a898247a21ae918fa952a993fbe407ebc49e404)) +* **db:** repair s3 asset paths missing default-storage leading slash ([#10243](https://github.com/windmill-labs/windmill/issues/10243)) ([555c751](https://github.com/windmill-labs/windmill/commit/555c751016fea4087be09fa2520af331cc872bfc)) +* **frontend:** curl fallback for +Variable/+Resource in bash sandbox mode ([#10235](https://github.com/windmill-labs/windmill/issues/10235)) ([28966bd](https://github.com/windmill-labs/windmill/commit/28966bdbf190ac461aa5259e04d6c15cf699fd94)) +* **frontend:** limit compare & deploy rows to the active direction ([#10234](https://github.com/windmill-labs/windmill/issues/10234)) ([7ac27c1](https://github.com/windmill-labs/windmill/commit/7ac27c1ef240729fc38d5fa506b96bf60be74e25)) +* **frontend:** prevent browser back-swipe navigation over monaco editors ([#10229](https://github.com/windmill-labs/windmill/issues/10229)) ([b0bf256](https://github.com/windmill-labs/windmill/commit/b0bf25683ba3aac3e94bbe9d6bbd6e22545e734e)) +* **parsers:** keep s3 asset path suffix verbatim to preserve storage distinction ([#10241](https://github.com/windmill-labs/windmill/issues/10241)) ([7fb8a2e](https://github.com/windmill-labs/windmill/commit/7fb8a2e390cef3cd01a33c89ee55d3f53cb1e20f)) +* **parser:** spurious pg arg inferred from placeholders in comments ([#10226](https://github.com/windmill-labs/windmill/issues/10226)) ([d24e176](https://github.com/windmill-labs/windmill/commit/d24e1768163c10fbbc144bd1126457e42d282e37)) +* **pg:** actionable error when s3object input exceeds jsonb 256MB cap ([#10228](https://github.com/windmill-labs/windmill/issues/10228)) ([6e42633](https://github.com/windmill-labs/windmill/commit/6e4263364325e4d4616697c6da601bbc887ff43c)) +* **postgres-triggers:** enforce resource-path scopes on ancillary routes ([#10222](https://github.com/windmill-labs/windmill/issues/10222)) ([39058c0](https://github.com/windmill-labs/windmill/commit/39058c0a01c35cf553daae3af37c6d326a7c8763)) +* return to parent workspace when a fork is deleted remotely ([#9898](https://github.com/windmill-labs/windmill/issues/9898)) ([32994df](https://github.com/windmill-labs/windmill/commit/32994df427e6df6747caf1ba6371512d5a5be5c2)) +* **s3:** support instance-policy credentials in object storage tests ([#10238](https://github.com/windmill-labs/windmill/issues/10238)) ([ec63244](https://github.com/windmill-labs/windmill/commit/ec6324409d6c0e9ddcda5d110ac570ca79fe9e38)) +* **triggers:** apply scope-path filtering to list and fix update scope check ([#10220](https://github.com/windmill-labs/windmill/issues/10220)) ([adc555d](https://github.com/windmill-labs/windmill/commit/adc555d1722cb59aa5b88503b3105357d76f5b71)) +* **worker:** mount /dev/shm as tmpfs in the Docker v2 nsjail sandbox ([#10240](https://github.com/windmill-labs/windmill/issues/10240)) ([2caee41](https://github.com/windmill-labs/windmill/commit/2caee41fdfe1e69010a1f4544d45aaa5db49f590)) + +## [1.764.0](https://github.com/windmill-labs/windmill/compare/v1.763.0...v1.764.0) (2026-07-20) + + +### Features + +* **ai:** add npm package search tool to global chat ([#10204](https://github.com/windmill-labs/windmill/issues/10204)) ([0593ff7](https://github.com/windmill-labs/windmill/commit/0593ff7d7d598af0b9f52d08bb22147e3a83e6e9)) +* **ai:** expose get_db_schema tool in global chat ([#10207](https://github.com/windmill-labs/windmill/issues/10207)) ([f4308cf](https://github.com/windmill-labs/windmill/commit/f4308cf033b75e9a69c1e21041ff97f3bfa1787c)) +* **ai:** extract prompt cache token usage from OpenAI and Azure providers ([#10214](https://github.com/windmill-labs/windmill/issues/10214)) ([2b58df5](https://github.com/windmill-labs/windmill/commit/2b58df57fc5bae08155ffd7198643ed739e6a463)) +* **ai:** live web-search source list on chat tool cards ([#10210](https://github.com/windmill-labs/windmill/issues/10210)) ([542a484](https://github.com/windmill-labs/windmill/commit/542a4842a317a6607de973b4567bdb27dc088664)) +* cap queued jobs per concurrency key on cloud ([#10197](https://github.com/windmill-labs/windmill/issues/10197)) ([71f2d47](https://github.com/windmill-labs/windmill/commit/71f2d47cb4ff331cbcc7843bf0704ddb63cb77db)) +* detect server-handled git-sync so CLI picks git push vs wmill sync push ([#10201](https://github.com/windmill-labs/windmill/issues/10201)) ([b070f56](https://github.com/windmill-labs/windmill/commit/b070f56c5e6109c219a53cbb9628e28be4397fb7)) +* expose windmill api endpoint catalog to global ai chat ([#10199](https://github.com/windmill-labs/windmill/issues/10199)) ([83a354f](https://github.com/windmill-labs/windmill/commit/83a354f831cbc1821f2316e6d4e1c7f7a2cd7cdb)) +* **jobs:** cap total queued jobs per workspace on cloud ([#10218](https://github.com/windmill-labs/windmill/issues/10218)) ([ddec2ab](https://github.com/windmill-labs/windmill/commit/ddec2abbb3f56fc44d4c75fde60f00fb1c6373bb)) +* **sessions:** live DOM access for the raw-app preview in AI sessions ([#10129](https://github.com/windmill-labs/windmill/issues/10129)) ([b448af1](https://github.com/windmill-labs/windmill/commit/b448af1da7f6a351973724721e52682259212292)) +* **telemetry:** generic feature-usage telemetry with AI session metrics ([#10200](https://github.com/windmill-labs/windmill/issues/10200)) ([11fda89](https://github.com/windmill-labs/windmill/commit/11fda89b520cc8d1ce30cdba36bd10355cf025cc)) + + +### Bug Fixes + +* **db:** grant schema usage and re-run windmill role grants ([#10212](https://github.com/windmill-labs/windmill/issues/10212)) ([38ceae1](https://github.com/windmill-labs/windmill/commit/38ceae1a098ae165c75f43aa9049b5afafdd3bac)) +* **forks:** show workspace settings link in sidebar for fork creators ([#10216](https://github.com/windmill-labs/windmill/issues/10216)) ([11bb37d](https://github.com/windmill-labs/windmill/commit/11bb37d7ca7fbd50cc66905f41851226a48e57fb)) +* **git-sync:** avoid percent-encoded colon in git-sync hub script path ([#10213](https://github.com/windmill-labs/windmill/issues/10213)) ([87be041](https://github.com/windmill-labs/windmill/commit/87be041c091dd7fdd8c6be907a9b2894779f9ac0)) +* **inputs:** add ownership check to saved-input args read endpoint ([#10194](https://github.com/windmill-labs/windmill/issues/10194)) ([f32d770](https://github.com/windmill-labs/windmill/commit/f32d7702bcb592a70f5cf7e01f9e830047645240)) +* **schedules:** apply scope-path filtering to schedule list endpoints ([#10192](https://github.com/windmill-labs/windmill/issues/10192)) ([70359e3](https://github.com/windmill-labs/windmill/commit/70359e3a76d2e1fa5994acce4909753410c80664)) +* **security:** enforce token scope filtering on folder list endpoints ([#10193](https://github.com/windmill-labs/windmill/issues/10193)) ([b5e69ff](https://github.com/windmill-labs/windmill/commit/b5e69ffba6afd49d147e9b5e88e2d7020c09142f)) +* **sessions:** session bar badge readouts, job persistence, refresh bounce ([#10217](https://github.com/windmill-labs/windmill/issues/10217)) ([c8870d3](https://github.com/windmill-labs/windmill/commit/c8870d36aed2283bfa14419cd0bb346083f27b9c)) +* steer ai chat away from draft-blind api catalog reads and runs ([#10202](https://github.com/windmill-labs/windmill/issues/10202)) ([0e04bc6](https://github.com/windmill-labs/windmill/commit/0e04bc6991486b2b2b05b13feed5b083e403ac80)) + +## [1.763.0](https://github.com/windmill-labs/windmill/compare/v1.762.2...v1.763.0) (2026-07-19) + + +### Features + +* **ai:** gate data pipelines in sessions behind a dev flag ([#10178](https://github.com/windmill-labs/windmill/issues/10178)) ([42da20a](https://github.com/windmill-labs/windmill/commit/42da20ae9743eab2f86b8f9c261ee567f8bbb0c5)) + + +### Bug Fixes + +* **cli:** stop emitting has_on_behalf_of/has_permissioned_as: false on pull ([#10188](https://github.com/windmill-labs/windmill/issues/10188)) ([2d77e74](https://github.com/windmill-labs/windmill/commit/2d77e742077b71be192a04ed9e85d4a024dc7f91)) + +## [1.762.2](https://github.com/windmill-labs/windmill/compare/v1.762.1...v1.762.2) (2026-07-18) + + +### Bug Fixes + +* **scripts:** populate auto_kind from draft JSON for draft-only scripts ([#10183](https://github.com/windmill-labs/windmill/issues/10183)) ([cba5f0d](https://github.com/windmill-labs/windmill/commit/cba5f0d8a8c2261a190d0f10396a6ad6aed89366)) + +## [1.762.1](https://github.com/windmill-labs/windmill/compare/v1.762.0...v1.762.1) (2026-07-17) + + +### Bug Fixes + +* **schedules:** stop disabling schedules on transient push errors ([#10179](https://github.com/windmill-labs/windmill/issues/10179)) ([c82056c](https://github.com/windmill-labs/windmill/commit/c82056cfde93ab0e26cf82aa4f7650c060565542)) +* **worker:** gate ansible socket-dir name check to unix ([#10180](https://github.com/windmill-labs/windmill/issues/10180)) ([203f6c6](https://github.com/windmill-labs/windmill/commit/203f6c69dd24fc055a4bf604315aa757f35d0565)) + +## [1.762.0](https://github.com/windmill-labs/windmill/compare/v1.761.0...v1.762.0) (2026-07-17) + + +### Features + +* **ai-chat:** image attachments and agent raw-app screenshots ([#10130](https://github.com/windmill-labs/windmill/issues/10130)) ([7a139ab](https://github.com/windmill-labs/windmill/commit/7a139ab23e034321554d0c85540d54251e61cc72)) +* **forks:** let a fork's creator manage developers on it without being an admin ([#10166](https://github.com/windmill-labs/windmill/issues/10166)) ([97f4477](https://github.com/windmill-labs/windmill/commit/97f44770698eb3dd9e296012360b7d9a4492e28d)) +* **otel-tracing-proxy:** trust internal endpoints with untrusted CAs ([#10139](https://github.com/windmill-labs/windmill/issues/10139)) ([396fb1c](https://github.com/windmill-labs/windmill/commit/396fb1c4752b2b93ef0ec9d0c9c6be05aa403eef)) +* **worker-tags:** add `*` fork marker to workspace-scoped custom tags ([#10177](https://github.com/windmill-labs/windmill/issues/10177)) ([2ff5a91](https://github.com/windmill-labs/windmill/commit/2ff5a918d54074bc340b6e92222e9f5bb4369ba0)) + + +### Bug Fixes + +* **ai:** unbreak session chat compaction for Anthropic models ([#10171](https://github.com/windmill-labs/windmill/issues/10171)) ([a9fc9f7](https://github.com/windmill-labs/windmill/commit/a9fc9f74b2c5b280cc087f0bfbac50112081c5e8)) +* **ansible:** keep persistent-connection socket path under the AF_UNIX limit ([#10167](https://github.com/windmill-labs/windmill/issues/10167)) ([be57dd9](https://github.com/windmill-labs/windmill/commit/be57dd91e40551ff0d6fbbeccd97e2b98589c5b9)) +* **extra:** make the extra container runnable as a non-root UID ([#10173](https://github.com/windmill-labs/windmill/issues/10173)) ([dcb9e40](https://github.com/windmill-labs/windmill/commit/dcb9e40ea237c3744611ac6e83ff5ecc6528e697)) +* **flows:** make updateFlow body path optional so AI can update flows ([#10176](https://github.com/windmill-labs/windmill/issues/10176)) ([7d2c5ce](https://github.com/windmill-labs/windmill/commit/7d2c5ceb0fb8609aa58a1e28e61ae8ce86e48208)) +* **frontend:** scope session pipeline trigger editors to the session workspace ([#10032](https://github.com/windmill-labs/windmill/issues/10032)) ([8828341](https://github.com/windmill-labs/windmill/commit/8828341a2b0912cbf44d6c6d81b4f4e30c5fe631)) +* **mcp:** apply token scopes consistently across mcp endpoint tools ([#10162](https://github.com/windmill-labs/windmill/issues/10162)) ([ae3d9ce](https://github.com/windmill-labs/windmill/commit/ae3d9ce2c031beeafc4e8156e0c85f163647af0f)) +* parse all names in grouped go param declarations ([#10165](https://github.com/windmill-labs/windmill/issues/10165)) ([d0aa7dc](https://github.com/windmill-labs/windmill/commit/d0aa7dca13aedfec90b7e023954cc824dafa3d34)) +* **schedules:** re-arm enabled schedules left with no queued occurrence ([#10174](https://github.com/windmill-labs/windmill/issues/10174)) ([9762089](https://github.com/windmill-labs/windmill/commit/9762089fcbdb33dbf453d22342661e5857b2ad7e)) + + +### Performance Improvements + +* lazy-load session preview editor views for snappy AI sessions switch ([#10172](https://github.com/windmill-labs/windmill/issues/10172)) ([1edee8a](https://github.com/windmill-labs/windmill/commit/1edee8aa34503d2892c1ef7b1deabf99e4b528b4)) + +## [1.761.0](https://github.com/windmill-labs/windmill/compare/v1.760.1...v1.761.0) (2026-07-16) + + +### Features + +* **ai-sessions:** CRUD markdown artifacts in sessions ([#10046](https://github.com/windmill-labs/windmill/issues/10046)) ([0ea5705](https://github.com/windmill-labs/windmill/commit/0ea570570e65abf64b3b171e592a8dd3eea0b105)) +* **alerts:** include disk total and top consumers in low-disk alert ([#10144](https://github.com/windmill-labs/windmill/issues/10144)) ([4e0fd4d](https://github.com/windmill-labs/windmill/commit/4e0fd4db5589f91426ca025184da7b977d5a16d6)) +* automatic git-to-windmill sync (polling, webhooks, in-app PRs + checks) ([#9552](https://github.com/windmill-labs/windmill/issues/9552)) ([51d8db6](https://github.com/windmill-labs/windmill/commit/51d8db6602bc192879e50b54a5fe6b6b8beb63f4)) +* display openai reasoning summaries in ai chat ([#10147](https://github.com/windmill-labs/windmill/issues/10147)) ([4ee1d32](https://github.com/windmill-labs/windmill/commit/4ee1d32101eb527ef707c3c4af456892f0a059ab)) +* **forks:** add "Hide unchanged drafts" toggle to fork deploy-draft tab ([#10022](https://github.com/windmill-labs/windmill/issues/10022)) ([4fc3f30](https://github.com/windmill-labs/windmill/commit/4fc3f304c632eda15876c28cb6f473a799a41f48)) +* **frontend:** flatten workspace pickers, whole-tab picker trigger ([#10145](https://github.com/windmill-labs/windmill/issues/10145)) ([7fda6a0](https://github.com/windmill-labs/windmill/commit/7fda6a05345d14f62dc2124d682650d50eecbe98)) + + +### Bug Fixes + +* **ai:** show the question in askUserQuestion tool-call labels ([#10153](https://github.com/windmill-labs/windmill/issues/10153)) ([fa03984](https://github.com/windmill-labs/windmill/commit/fa03984a14bb71f0ea2a27bfa23339a9d0d8e86b)) +* **alerts:** identify server replica in low-disk alert + per-host dedup tag ([#10143](https://github.com/windmill-labs/windmill/issues/10143)) ([3bd9f05](https://github.com/windmill-labs/windmill/commit/3bd9f0593866740935f79c8b2c1cb92829059105)) +* **frontend:** sanitize job result markup, gate it on unsandboxed public apps ([#10127](https://github.com/windmill-labs/windmill/issues/10127)) ([7b813d1](https://github.com/windmill-labs/windmill/commit/7b813d1f74baef200d9f1387e02925083b79b265)) +* **frontend:** show friendly draft path for draft-only items in pickers ([#10136](https://github.com/windmill-labs/windmill/issues/10136)) ([568dbbe](https://github.com/windmill-labs/windmill/commit/568dbbee852d59986581b0d16f5ac3e06d6bad6a)) +* **frontend:** surface real tool call errors in AI chat ([#10146](https://github.com/windmill-labs/windmill/issues/10146)) ([0694b84](https://github.com/windmill-labs/windmill/commit/0694b84da7bbb9404fd386e3ce109a574a0e9c93)) +* heartbeat job ping during s3object materialization in SQL executors ([#10152](https://github.com/windmill-labs/windmill/issues/10152)) ([7d5009e](https://github.com/windmill-labs/windmill/commit/7d5009e3928bb200150eb998ed0b3016c3395987)) +* **mcp:** push granular scope patterns into SQL so scoped scripts/flows aren't truncated ([#10140](https://github.com/windmill-labs/windmill/issues/10140)) ([91d6606](https://github.com/windmill-labs/windmill/commit/91d6606868d0e9c5f78ab48a592ef95ffaeeca61)) +* **migrations:** grant zombie_job_counter to windmill roles ([#10159](https://github.com/windmill-labs/windmill/issues/10159)) ([0e547ad](https://github.com/windmill-labs/windmill/commit/0e547adf23b615f0caccc88eb9101bb0604bb51b)) +* **raw-apps:** full code ui builder improvements ([c55ac53](https://github.com/windmill-labs/windmill/commit/c55ac5326fbb4532f22b7250a58b69396a4c868a)) +* **raw-apps:** prevent and surface the silent blank screen from an unmounted #root ([#10150](https://github.com/windmill-labs/windmill/issues/10150)) ([24750e6](https://github.com/windmill-labs/windmill/commit/24750e6ef1a1a975e0063df6809774f913b5d01b)) +* **self-host:** unbreak self-hosted Caddy after the caddy-l4 syntax change ([#10156](https://github.com/windmill-labs/windmill/issues/10156)) ([2f6c35b](https://github.com/windmill-labs/windmill/commit/2f6c35b15bf70f17eb8ff08281e0ecf8313e7b35)) + +## [1.760.1](https://github.com/windmill-labs/windmill/compare/v1.760.0...v1.760.1) (2026-07-15) + + +### Bug Fixes + +* **apps:** honor presigned S3 signature on app display/preview routes ([#10141](https://github.com/windmill-labs/windmill/issues/10141)) ([8c725d9](https://github.com/windmill-labs/windmill/commit/8c725d9e44e38bf3b35f41b847cdf8745e07942e)) + +## [1.760.0](https://github.com/windmill-labs/windmill/compare/v1.759.0...v1.760.0) (2026-07-15) + + +### Features + +* **nsjail:** make python/ansible rlimit_as configurable per worker (GIT-921) ([#10138](https://github.com/windmill-labs/windmill/issues/10138)) ([1787201](https://github.com/windmill-labs/windmill/commit/17872018cc037e699b1f6e1589d0ad05e0883cca)) + + +### Bug Fixes + +* **ai:** disable redirects on worker AI provider client (GHSA-5q4v) ([#10122](https://github.com/windmill-labs/windmill/issues/10122)) ([27ead8d](https://github.com/windmill-labs/windmill/commit/27ead8d0848cceacaf0c49fed0e8896472b851e3)) +* **ai:** stop sending the AI agent system prompt twice for OpenAI ([#10126](https://github.com/windmill-labs/windmill/issues/10126)) ([8bfe5c9](https://github.com/windmill-labs/windmill/commit/8bfe5c93404ba3f137394f16d0571a06d891dc3b)) +* **apps:** invalidate cached app policy on change or deletion (GHSA-r5v4-cxh9-7qhq) ([#10121](https://github.com/windmill-labs/windmill/issues/10121)) ([f7eb5c4](https://github.com/windmill-labs/windmill/commit/f7eb5c460d78792c24297e20cb062638522a4f68)) +* **bash:** normalize CRLF line endings before running scripts ([#10131](https://github.com/windmill-labs/windmill/issues/10131)) ([6407d9f](https://github.com/windmill-labs/windmill/commit/6407d9ff5ce51e71ff8b8fc503d89a2bdc2e1761)) +* **cli-image:** patch fixable CRITICAL CVEs in windmill-cli image (GIT-922) ([#10135](https://github.com/windmill-labs/windmill/issues/10135)) ([5626768](https://github.com/windmill-labs/windmill/commit/56267684718944ef4d4ecd3b610bad81132e1990)) +* **frontend:** graceful small-screen timeframe picker on the runs page ([#10073](https://github.com/windmill-labs/windmill/issues/10073)) ([af177ce](https://github.com/windmill-labs/windmill/commit/af177cefe07e6037e33cc90757088a98fb63a49f)) +* **frontend:** keep session-exit URL clean by syncing new_draft strip with the router ([#10101](https://github.com/windmill-labs/windmill/issues/10101)) ([9705d60](https://github.com/windmill-labs/windmill/commit/9705d602848966f850613233d6e23b73a753c259)) +* **frontend:** only carry custom-tag overrides on 'Run again' ([#10137](https://github.com/windmill-labs/windmill/issues/10137)) ([bd3adc9](https://github.com/windmill-labs/windmill/commit/bd3adc9781d8e77928c9feb03d0e05b63a1aaf7c)) +* **frontend:** treat a displaced draft save as superseded, not failed ([#10094](https://github.com/windmill-labs/windmill/issues/10094)) ([2fe999f](https://github.com/windmill-labs/windmill/commit/2fe999f66cd15acd81850f970ada31e9892abff2)) +* reject git URL fragment/query SSRF bypass (GHSA-p5cj-8cfh-mjv6) ([#10120](https://github.com/windmill-labs/windmill/issues/10120)) ([73c8d7f](https://github.com/windmill-labs/windmill/commit/73c8d7f08ad55cd2323d1bb9438f00a8ac30e046)) +* **security:** enforce variables:write scope on resource-delete var cascade (GHSA-xmr2-98m6-cjf7) ([#10123](https://github.com/windmill-labs/windmill/issues/10123)) ([188647a](https://github.com/windmill-labs/windmill/commit/188647a942a0cb496d63c71bf3e0f7a3136ec217)) + ## [1.759.0](https://github.com/windmill-labs/windmill/compare/v1.758.0...v1.759.0) (2026-07-15) diff --git a/Caddyfile b/Caddyfile index 07496bdbfb..3008c93855 100644 --- a/Caddyfile +++ b/Caddyfile @@ -16,8 +16,12 @@ # site, silently disabling the HTTP proxy while the :25 layer4 listener stays up. bind {$ADDRESS:0.0.0.0 ::} - # Extra services: LSP, Multiplayer, Debugger (windmill_extra gateway) - reverse_proxy /ws/* /ws_mp/* /ws_debug/* http://windmill_extra:3000 + # Extra services: LSP, Multiplayer, Debugger (windmill_extra gateway). + # reverse_proxy only reads its first argument as a matcher, so listing + # several paths inline turns the rest into upstream addresses. The paths + # have to go through a named matcher. + @extra path /ws/* /ws_mp/* /ws_debug/* + reverse_proxy @extra http://windmill_extra:3000 # Search indexer, Enterprise Edition (windmill_indexer:8002) # reverse_proxy /api/srch/* http://windmill_indexer:8002 diff --git a/Dockerfile b/Dockerfile index 83bf4c8513..b0b429803e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,8 @@ COPY /python-client/docs/ /frontend/static/pydocs/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" ARG VITE_BASE_URL "" +# Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js. +ARG WM_BUILD_VERSION="" # Read more about macro in docker/dev.nu # -- MACRO-SPREAD-WASM-PARSER-DEV-ONLY -- # RUN npm run build diff --git a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts index e553d15f56..8fe28d022a 100644 --- a/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/global/globalEvalRunner.ts @@ -3,7 +3,7 @@ import { tmpdir } from "os"; import { join } from "path"; import type { AIProvider } from "$lib/gen/types.gen"; import { - globalTools, + globalToolsFor, prepareGlobalSystemMessage, prepareGlobalUserMessage, } from "../../../../../frontend/src/lib/components/copilot/chat/global/core"; @@ -43,6 +43,67 @@ const LIVE_EDITOR_ITEM_KINDS = { app: "raw_app", } as const; +// SessionArtifactsStore can't run here (bun has no IndexedDB, nor the compiled $state runes), +// so mirror only its tool-facing shape; its own logic (scoping, race guard) is unit-tested. +const EVAL_SESSION_ID = "eval-session"; +function createEvalArtifactHelpers() { + const items = new Map>(); + let seq = 0; + const store = { + create: async (sessionId: string, input: Record) => { + const now = seq++; + const artifact = { + id: `eval-artifact-${now}`, + sessionId, + chatId: input.chatId, + kind: input.kind ?? "md", + name: input.name, + content: input.content, + createdAt: now, + updatedAt: now, + }; + items.set(artifact.id, artifact); + return artifact; + }, + get: async (id: string) => items.get(id), + update: async ( + id: string, + input: Record, + opts?: { sessionId?: string }, + ) => { + const existing = items.get(id); + if (!existing) return undefined; + if ( + opts?.sessionId !== undefined && + existing.sessionId !== opts.sessionId + ) + return undefined; + const updated = { + ...existing, + name: input.name ?? existing.name, + content: input.content ?? existing.content, + updatedAt: seq++, + }; + items.set(id, updated); + return updated; + }, + remove: async (id: string) => { + items.delete(id); + }, + listForSession: async (sessionId: string) => + [...items.values()].filter((a) => a.sessionId === sessionId), + }; + return { + helpers: { + artifacts: store, + sessionId: EVAL_SESSION_ID, + getChatId: () => "eval-chat", + openArtifact: () => {}, + }, + snapshot: () => [...items.values()], + }; +} + export interface GlobalLiveEditorDraftFixture { type: keyof typeof LIVE_EDITOR_ITEM_KINDS; storagePath?: string; @@ -80,6 +141,8 @@ export interface GlobalEvalOptions { workspaceFixtures?: BenchmarkWorkspaceRunnables; liveEditorDrafts?: GlobalLiveEditorDraftFixture[]; user?: GlobalUserFixture; + // Emulate a session chat (preview tools + session prompt); default false = standalone baseline. + sessionChat?: boolean; model?: string; maxIterations?: number; provider?: AIProvider; @@ -98,7 +161,10 @@ export async function runGlobalEval( (await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-"))); clearGlobalDrafts(workspaceRoot); - registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {}); + registerBenchmarkWorkspaceRunnables( + workspaceRoot, + options.workspaceFixtures ?? {}, + ); seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []); try { @@ -107,18 +173,25 @@ export async function runGlobalEval( process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1"; // Pass the seeded identity straight to the prompt builder rather than mutating // the process-global `userStore`, so concurrent cases never race on it. + const evalArtifacts = createEvalArtifactHelpers(); const rawResult = await runEval({ userPrompt, - systemMessage: prepareGlobalSystemMessage(undefined, { user: options.user }), + systemMessage: prepareGlobalSystemMessage(undefined, { + user: options.user, + previewTools: options.sessionChat ?? false, + }), userMessage: prepareGlobalUserMessage( userPrompt, [], injectActiveEditorContext ? { workspace: workspaceRoot } : {}, ), - tools: getGlobalEvalTools(), - helpers: {}, + tools: getGlobalEvalTools(options.sessionChat ?? false), + helpers: evalArtifacts.helpers, apiKey, - getOutput: () => collectGlobalDraftState(workspaceRoot), + getOutput: async () => ({ + ...(await collectGlobalDraftState(workspaceRoot)), + artifacts: evalArtifacts.snapshot(), + }), onAssistantMessageStart: options.runContext?.onAssistantMessageStart, onAssistantToken: options.runContext?.onAssistantChunk, onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd, @@ -213,10 +286,15 @@ function clearLiveEditorDrafts( } } -function getGlobalEvalTools(): ProductionTool<{}>[] { +// Gate session-preview tools on sessionChat, as production's globalToolsFor does. +function getGlobalEvalTools(sessionChat: boolean): ProductionTool<{}>[] { const disableSearchApp = process.env[DISABLE_SEARCH_APP_ENV] === "1"; - return (globalTools as ProductionTool<{}>[]) - .filter((tool) => !(disableSearchApp && tool.def.function.name === "search_app")) + return ( + globalToolsFor({ sessionPreview: sessionChat }) as ProductionTool<{}>[] + ) + .filter( + (tool) => !(disableSearchApp && tool.def.function.name === "search_app"), + ) .map((tool) => { if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) { return tool; diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index f9023449a2..64cacabd58 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -10,6 +10,7 @@ import type { import type { DataTableTables, DataTableTableSchema, + EndpointTool, GetDraftForUserResponse, GetOwnDraftResponse, ListDraftsResponse, @@ -303,13 +304,20 @@ export function getBenchmarkJobLogs(workspace: string, jobId: string): string { */ const benchmarkDrafts = new Map< string, - { workspace: string; kind: UserDraftItemKind; path: string; value: unknown } + { workspace: string; kind: UserDraftItemKind; path: string; value: unknown; createdAt: string } >() -// Fixed timestamp so artifacts stay deterministic. No eval simulates a -// concurrent writer, so every save is accepted and the conflict branch is -// never taken — the syncer just records this as its `last_sync` baseline. -const BENCHMARK_DRAFT_TIMESTAMP = '1970-01-01T00:00:00.000Z' +// Counter-based timestamps: deterministic run-to-run (same event order → same +// values) but MONOTONIC per update, because production bumps a draft row's +// created_at on every upsert and the diff snapshot cache keys patch reuse on +// it — a fixed timestamp would serve stale patches after an edit. No eval +// simulates a concurrent writer, so every save is accepted and the conflict +// branch is never taken. +let benchmarkDraftClock = 0 +function nextBenchmarkDraftTimestamp(): string { + benchmarkDraftClock += 1 + return new Date(benchmarkDraftClock * 1000).toISOString() +} function benchmarkDraftKey(workspace: string, kind: string, path: string): string { return `${workspace}::${kind}::${path}` @@ -340,7 +348,8 @@ export function seedBenchmarkDraft( workspace, kind, path, - value + value, + createdAt: nextBenchmarkDraftTimestamp() }) } @@ -353,6 +362,7 @@ export function updateBenchmarkDraft(input: { }): UpdateDraftResponse { const key = benchmarkDraftKey(input.workspace, input.kind, input.path) const value = input.requestBody?.value + const createdAt = nextBenchmarkDraftTimestamp() if (value == null) { benchmarkDrafts.delete(key) } else { @@ -360,10 +370,11 @@ export function updateBenchmarkDraft(input: { workspace: input.workspace, kind: input.kind, path: input.path, - value + value, + createdAt }) } - return { status: 'saved', current_timestamp: BENCHMARK_DRAFT_TIMESTAMP } + return { status: 'saved', current_timestamp: createdAt } } /** Mirror `DraftService.getDraftForUser`: 404-shaped throw when absent so the @@ -377,7 +388,7 @@ export function getBenchmarkDraftForUser(input: { if (!entry) { throw Object.assign(new Error(`no draft for "${input.path}"`), { status: 404 }) } - return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } + return { value: entry.value, created_at: entry.createdAt } } /** Mirror `DraftService.getOwnDraft`: `null` (200) when absent — unlike @@ -391,7 +402,18 @@ export function getBenchmarkOwnDraft(input: { if (!entry) { return null } - return { value: entry.value, created_at: BENCHMARK_DRAFT_TIMESTAMP } + return { value: entry.value, created_at: entry.createdAt } +} + +/** Whether a deployed benchmark item exists for a draft row's kind+path — + * drives `draft_only`, which production computes against the deployed tables. */ +function benchmarkDeployedExists(workspace: string, kind: UserDraftItemKind, path: string): boolean { + if (kind === 'script') return Boolean(getBenchmarkScriptByPath(workspace, path)) + if (kind === 'flow') return Boolean(getBenchmarkFlowByPath(workspace, path)) + if (kind === 'app' || kind === 'raw_app') return Boolean(getBenchmarkAppByPath(workspace, path)) + // Drawer kinds (variables/resources/schedules/triggers) have no deployed + // benchmark stores today. + return false } /** Mirror `DraftService.listDrafts`: metadata rows (no value) for a workspace. */ @@ -402,9 +424,9 @@ export function listBenchmarkDrafts(workspace: string): ListDraftsResponse { kind: entry.kind, path: entry.path, summary: (entry.value as { summary?: string } | null)?.summary, - draft_only: true, + draft_only: !benchmarkDeployedExists(workspace, entry.kind, entry.path), legacy_draft: false, - created_at: BENCHMARK_DRAFT_TIMESTAMP + created_at: entry.createdAt })) } @@ -694,3 +716,138 @@ function buildBenchmarkApp(app: BenchmarkWorkspaceApp): AppWithLastVersion { raw_app: true } } + +// ============= API endpoint catalog (McpService.listMcpTools + raw fetch) ============= +// The global chat's API catalog tools list endpoints via McpService and execute +// them with a plain relative fetch('/api/...'), which has no meaning in the +// vitest environment. A representative slice of the real catalog is served here, +// and `handleBenchmarkApiFetch` answers the executed calls. + +const BENCHMARK_MCP_TOOLS: EndpointTool[] = [ + { + name: 'listWorkers', + description: 'List workers', + instructions: 'List all workers with their last ping and job counts.', + path: '/workers/list', + method: 'GET', + query_params_schema: { + type: 'object', + properties: { page: { type: 'integer' }, per_page: { type: 'integer' } } + } + }, + { + name: 'listQueue', + description: 'List queued jobs', + instructions: '', + path: '/w/{workspace}/jobs/queue/list', + method: 'GET', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' } }, + required: ['workspace'] + } + }, + { + name: 'runScriptByPath', + description: 'Run the deployed version of a script by path', + instructions: '', + path: '/w/{workspace}/jobs/run/p/{path}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, path: { type: 'string' } }, + required: ['workspace', 'path'] + }, + body_schema: { type: 'object', properties: {} } + }, + { + name: 'runFlowByPath', + description: 'Run the deployed version of a flow by path', + instructions: '', + path: '/w/{workspace}/jobs/run/f/{path}', + method: 'POST', + path_params_schema: { + type: 'object', + properties: { workspace: { type: 'string' }, path: { type: 'string' } }, + required: ['workspace', 'path'] + }, + body_schema: { type: 'object', properties: {} } + }, + // Draft-covered endpoints, present so steering cases exercise the guard the + // way production does (hidden from search, refused at call time). + { + name: 'getScriptByPath', + description: 'Get a script by path', + instructions: '', + path: '/w/{workspace}/scripts/get/p/{path}', + method: 'GET' + }, + { + name: 'createFlow', + description: 'Create a flow', + instructions: '', + path: '/w/{workspace}/flows/create', + method: 'POST' + }, + { + name: 'deleteSchedule', + description: 'Delete a schedule', + instructions: '', + path: '/w/{workspace}/schedules/delete/{path}', + method: 'DELETE' + }, + { + name: 'getVariable', + description: 'Get a variable', + instructions: '', + path: '/w/{workspace}/variables/get/{path}', + method: 'GET' + } +] + +export function listBenchmarkMcpTools(): EndpointTool[] { + return BENCHMARK_MCP_TOOLS +} + +const BENCHMARK_WORKERS = [ + { + worker: 'wk-benchmark-1', + worker_instance: 'benchmark-host', + last_ping: 2, + started_at: BENCHMARK_TIMESTAMP, + jobs_executed: 42, + custom_tags: null, + worker_group: 'default', + wm_version: 'benchmark' + }, + { + worker: 'wk-benchmark-2', + worker_instance: 'benchmark-host', + last_ping: 5, + started_at: BENCHMARK_TIMESTAMP, + jobs_executed: 17, + custom_tags: null, + worker_group: 'default', + wm_version: 'benchmark' + } +] + +/** True when `handleBenchmarkApiFetch` has an answer for this `/api/...` url. + * Any other relative fetch must keep its normal (non-benchmark) behavior — + * intercepting it with a synthetic 404 sends the model into retry loops. */ +export function hasBenchmarkApiHandler(url: string): boolean { + const path = url.split('?')[0] + return path === '/api/workers/list' || /^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path) +} + +/** Answer a relative `/api/...` fetch issued by the API catalog executor. */ +export function handleBenchmarkApiFetch(url: string): Response { + const path = url.split('?')[0] + if (path === '/api/workers/list') { + return Response.json(BENCHMARK_WORKERS) + } + if (/^\/api\/w\/[^/]+\/jobs\/queue\/list$/.test(path)) { + return Response.json([]) + } + return Response.json({ error: `no benchmark handler for ${path}` }, { status: 404 }) +} diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 92e33414df..c07d104f47 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -3,6 +3,20 @@ import { expect, it, vi } from 'vitest' import { mkdir, writeFile } from 'fs/promises' // @ts-ignore - Node.js path import { dirname, resolve } from 'path' +import { handleBenchmarkApiFetch, hasBenchmarkApiHandler } from './mockBackend' + +// The API catalog executor issues relative fetch('/api/...') calls, which have +// no meaning in the vitest environment — serve the ones the benchmark handles. +// Every other relative fetch keeps its normal behavior (it fails the same way +// it does without this stub) so unrelated tools see an unchanged environment. +const ORIGINAL_FETCH = globalThis.fetch +globalThis.fetch = (async (input: unknown, init?: RequestInit) => { + const url = typeof input === 'string' ? input : ((input as Request | URL | null)?.url ?? '') + if (typeof url === 'string' && hasBenchmarkApiHandler(url)) { + return handleBenchmarkApiFetch(url) + } + return ORIGINAL_FETCH(input as Parameters[0], init) +}) as typeof fetch vi.mock('monaco-editor', () => ({ editor: {}, @@ -57,7 +71,8 @@ vi.mock('$lib/gen', async () => { runBenchmarkDatatableSql, runBenchmarkFlowByPath, runBenchmarkScriptPreview, - updateBenchmarkDraft + updateBenchmarkDraft, + listBenchmarkMcpTools } = await import('./mockBackend') function wrapService(target: T, overrides: Record): T { @@ -111,13 +126,32 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkScriptByPath(data.workspace, data.path)) : actual.ScriptService.existsScriptByPath(data), - getScriptByPath: async (data: { workspace: string; path: string }) => { + getScriptByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { if (hasBenchmarkWorkspace(data.workspace)) { const script = getBenchmarkScriptByPath(data.workspace, data.path) + // `getDraft` mirrors production's overlay: the row plus the caller's + // draft and a `no_deployed` marker (draft-only item). The diff tool + // reads through this shape — without it every draft looks absent. + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'script', path: data.path }) + : null if (!script) { - throw new Error(`Script "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`Script "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return script + return data.getDraft + ? { ...script, draft: draft?.value ?? undefined, no_deployed: false } + : script } return actual.ScriptService.getScriptByPath(data) }, @@ -151,13 +185,30 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? Boolean(getBenchmarkFlowByPath(data.workspace, data.path)) : actual.FlowService.existsFlowByPath(data), - getFlowByPath: async (data: { workspace: string; path: string }) => { + getFlowByPath: async (data: { workspace: string; path: string; getDraft?: boolean }) => { if (hasBenchmarkWorkspace(data.workspace)) { const flow = getBenchmarkFlowByPath(data.workspace, data.path) + // Mirror production's `getDraft` overlay (see getScriptByPath above). + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'flow', path: data.path }) + : null if (!flow) { - throw new Error(`Flow "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`Flow "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return flow + return data.getDraft + ? { ...flow, draft: draft?.value ?? undefined, no_deployed: false } + : flow } return actual.FlowService.getFlowByPath(data) }, @@ -299,6 +350,12 @@ vi.mock('$lib/gen', async () => { queryResourceTypes: async (data: { workspace: string }) => hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data) }), + McpService: wrapService(actual.McpService, { + listMcpTools: async (data: { workspace: string }) => + hasBenchmarkWorkspace(data.workspace) + ? listBenchmarkMcpTools() + : actual.McpService.listMcpTools(data) + }), VariableService: wrapService(actual.VariableService, { existsVariable: async (data: { workspace: string; path: string }) => hasBenchmarkWorkspace(data.workspace) ? false : actual.VariableService.existsVariable(data), @@ -320,13 +377,37 @@ vi.mock('$lib/gen', async () => { hasBenchmarkWorkspace(data.workspace) ? (listBenchmarkApps(data.workspace) ?? []) : actual.AppService.listApps(data), - getAppByPath: async (data: { workspace: string; path: string }) => { + getAppByPath: async (data: { + workspace: string + path: string + getDraft?: boolean + rawApp?: boolean + }) => { if (hasBenchmarkWorkspace(data.workspace)) { const app = getBenchmarkAppByPath(data.workspace, data.path) + // Mirror production's `getDraft` overlay (see getScriptByPath above). + // Benchmark app drafts live under the raw_app kind. + const draft = data.getDraft + ? getBenchmarkOwnDraft({ workspace: data.workspace, kind: 'raw_app', path: data.path }) + : null if (!app) { - throw new Error(`App "${data.path}" not found in benchmark workspace`) + if (data.getDraft && draft) { + return { + ...(draft.value as Record), + path: data.path, + raw_app: true, + draft: draft.value, + no_deployed: true + } + } + throw Object.assign( + new Error(`App "${data.path}" not found in benchmark workspace`), + { status: 404 } + ) } - return app + return data.getDraft + ? { ...app, draft: draft?.value ?? undefined, no_deployed: false } + : app } return actual.AppService.getAppByPath(data) } diff --git a/ai_evals/cases/flow.yaml b/ai_evals/cases/flow.yaml index a21ae81f17..fa2ca3bee7 100644 --- a/ai_evals/cases/flow.yaml +++ b/ai_evals/cases/flow.yaml @@ -480,3 +480,85 @@ judgeChecklist: - "the flow includes a final top-level step named `webhook_response`" - "`webhook_response` returns `ok: true` and the order summary" + +- id: flow-test17-implicit-schedule-intent + prompt: |- + I want this order processing flow to run on its own every morning at 07:30 UTC. + Set that up for me. Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_schedule + toolCallArgs: + - tool: create_schedule + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_schedule + field: schedule + stringIncludesAnyOf: + - 30 7 + - tool: create_schedule + field: timezone + stringIncludesAnyOf: + - UTC + skipJudge: true + judgeChecklist: + - "a schedule is created for the flow that runs daily at 07:30 UTC" + +- id: flow-test18-implicit-http-trigger-intent + prompt: |- + I need to be able to kick off this order processing flow by sending it an HTTP POST + from an external system, with no authentication. Use route path `ai-evals/order-processing-implicit`. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: path + stringStartsWithAnyOf: + - f/ + - u/ + stringMustNotStartWithAnyOf: + - schedules/ + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - http + - tool: create_trigger + field: config.http_method + stringIncludesAnyOf: + - post + - tool: create_trigger + field: config.authentication_method + stringIncludesAnyOf: + - none + - tool: create_trigger + field: config.route_path + stringIncludesAnyOf: + - ai-evals/order-processing-implicit + skipJudge: true + judgeChecklist: + - "an HTTP trigger is created for the flow that accepts unauthenticated POST requests" + +- id: flow-test19-implicit-email-trigger-intent + prompt: |- + Make this order processing flow run automatically whenever an email is received. + Do not ask me for the flow path. + initial: ai_evals/fixtures/frontend/flow/initial/scheduled_order_flow.json + toolExpect: + requiredToolsUsed: + - create_trigger + toolCallArgs: + - tool: create_trigger + field: kind + stringStartsWithAnyOf: + - email + skipJudge: true + judgeChecklist: + - "an email trigger (kind email) is created, or the user is told how to enable email triggering on the instance" diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 0f7f2a4717..72739d2778 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1050,11 +1050,44 @@ - opens the Kafka triggers page - does not write, deploy, or delete anything +- id: global-openpage7-compare-review + prompt: |- + Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it), then open the review page so I can look over the pending change and deploy it myself. + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + toolExpect: + requiredToolsUsed: + - write_script + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: open_page + field: page + stringIncludesAnyOf: + - compare + # The eval chat is untracked (no modified-items mask), so the model must scope + # the review by passing the item it changed explicitly — an omitted mask would + # preselect every pending change in the workspace. + - tool: open_page + field: items + stringIncludesAnyOf: + - f/evals/global/compare_review_demo + skipJudge: true + judgeChecklist: + - creates the script draft, then opens the Compare & Deploy review page instead of deploying itself + - preselects only the created script on the review page + - does not deploy or delete anything + - id: global-closepage1-close-runs-tab prompt: |- You just opened the runs page for me in the side panel. Close that tab, I'm done looking at it. runtime: maxTurns: 6 + sessionChat: true validate: draftCountExactly: 0 toolExpect: @@ -1451,9 +1484,15 @@ - deploy_workspace_item - delete_workspace_item judgeChecklist: + # A pipeline node is DECLARATIVE: triggers are declared by `-- on ` + # annotations (the trigger row is created separately) and a `-- materialize` + # output is a MANAGED write where the body is a bare SELECT that the runtime + # wraps in the create/replace. Do not expect a separate trigger config or a + # hand-written CREATE TABLE / INSERT — those would be wrong for a materialize node. - builds a data pipeline node as a script (not a flow) - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`) - - declares a schedule trigger and writes its output to a managed DuckLake table + - declares the schedule trigger with the `-- on schedule` annotation comment (this annotation is the correct and complete way a pipeline node binds a schedule; no separate trigger configuration is expected) + - declares the managed DuckLake output with `-- materialize ducklake://` and writes the body as a bare SELECT (materialize is a managed write, so the node correctly does NOT hand-write its own CREATE TABLE / INSERT) - leaves the result as an AI draft and does not deploy or save it - id: global-test-pipeline-two-node-chain @@ -1467,6 +1506,12 @@ maxTurns: 14 validate: draftCountAtLeast: 2 + requiredDrafts: + - type: script + pathStartsWith: f/evals/global/ + valueIncludes: + - pipeline + - ducklake forbiddenDrafts: - type: flow pathStartsWith: f/evals/global/ @@ -1478,12 +1523,65 @@ - deploy_workspace_item - delete_workspace_item judgeChecklist: + # Pipeline nodes are declarative: `-- on ` binds inputs/triggers and + # `-- materialize ducklake://
` is a managed write whose body is a bare + # SELECT. Do not expect hand-written CREATE TABLE / INSERT on a materialize node. - creates two data pipeline nodes as scripts (not a flow) in f/evals/global - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) - - the first ingests orders into a DuckLake table - - the second reads that same table and writes a daily rollup, wired to the first step's output asset + - the first ingests orders into a DuckLake table (a `-- materialize ducklake://
` output with a bare SELECT body is correct; no hand-written CREATE TABLE / INSERT is expected) + - the second reads that same table via `-- on ducklake://` and materializes a daily rollup table, wiring it to the first step's output asset - leaves both as AI drafts without deploying +- id: global-test-pipeline-complex-incremental + prompt: |- + Build a data pipeline in the `f/evals/global` folder for our web shop's + orders. It has three steps: + 1. On a schedule, ingest the raw order CSVs under `s3://raw/orders/` into a + managed DuckLake table. + 2. An incremental daily rollup: read that raw orders table and, on each run, + append just the current day's order count and total revenue into a second + DuckLake table. It should process one day at a time, not rebuild the whole + table every run. + 3. A final step that reads the daily rollup table and exports the latest data + as a Parquet file to `s3://reports/` for the BI team. + Wire each step to the previous step's output so they form one pipeline. Keep + everything as AI drafts — don't deploy or save. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 18 + validate: + draftCountAtLeast: 3 + requiredDrafts: + - type: script + pathStartsWith: f/evals/global/ + valueIncludes: + - pipeline + - ducklake + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + # Pipeline nodes are declarative: `-- on ` binds inputs/triggers, and a + # DuckLake `-- materialize` output is a managed write whose body is a bare SELECT + # (the runtime performs the create/replace/append/merge). Do not expect a + # separate trigger config or hand-written CREATE TABLE / INSERT on a + # materialize node. S3/Parquet output is NOT materialize: the body writes it. + - builds the pipeline as three independent scripts (not a flow) in f/evals/global + - every node carries the pipeline annotation in its own comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) + - step 1 binds a schedule with `-- on schedule` and declares a managed DuckLake output with `-- materialize ducklake://
` and a bare SELECT body (no separate trigger config or hand-written CREATE TABLE is expected) + - "step 2 is incremental: each run adds only that day's rows to a second DuckLake table rather than rebuilding the whole table every run (e.g. an `append` or `key=` merge materialize mode, not a full replace). Selecting the day via the `-- partitioned daily` + `{partition}` / `wm_partition(...)` idiom is the idiomatic form, but an equivalent current-day filter also satisfies this; a full-refresh/replace of the whole table does not" + - step 2 reads the same DuckLake table step 1 writes (via `-- on ducklake://`), wiring it to step 1's output asset + - step 3 reads the daily rollup table and exports it as a Parquet file to S3 + - does not misuse `-- materialize` for the S3 Parquet export (materialize is DuckLake-only; the S3 output is written by the script body, e.g. a DuckDB COPY or an SDK write) + - leaves all three nodes as AI drafts without deploying or saving + - id: global-path5-create-folder-then-draft prompt: |- Create a new shared folder called "analytics" for our data work, then draft a @@ -1509,3 +1607,273 @@ - creates a new shared folder named "analytics" via create_folder - drafts a script placed in that folder (f/analytics/...) returning an ISO timestamp - leaves the script as a draft only + +- id: global-artifact-plan-create + prompt: |- + I'm about to build a customer onboarding flow, but first I want a short written plan I can review and iterate on before any code. + Draft a markdown plan with a title, a one-sentence summary, and three or four bullet steps. + Keep it as something I can reopen and revise later — don't build the flow itself yet. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json + runtime: + maxTurns: 6 + sessionChat: true + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - create_artifact + forbiddenToolsUsed: + - write_flow + - write_script + - deploy_workspace_item + judgeChecklist: + - saves the plan as a markdown artifact via create_artifact rather than only replying inline + - the artifact content has a title, a one-line summary, and three or four bullet steps for onboarding + - does not create a flow or script draft yet + +- id: global-npm1-script-search-package + prompt: |- + Find a good npm package for parsing RSS/Atom feeds and use it to create a draft Bun script + at `f/evals/global/fetch_feed`. The script should take a string `url` input and return the + feed title along with the titles of the 5 most recent items. Tell me which package you + picked and link its documentation. Leave it as an AI draft only; do not deploy. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/fetch_feed + language: bun + valueIncludes: + - url + toolExpect: + requiredToolsUsed: + - search_npm_packages + - write_script + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - searches npm for a feed-parsing package before writing the code + - the draft Bun script imports the chosen npm package rather than hand-rolling an XML/RSS parser + - the script takes a url input and returns the feed title plus the recent item titles + - the reply names the chosen package and links its documentation + - the result stays as an AI draft and is not deployed + +- id: global-dbschema1-postgres-resource-tables + prompt: |- + I have a postgres resource at f/data/reports_pg in this workspace. + What tables does that database have? + initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - get_db_schema + forbiddenToolsUsed: + - write_script + - write_resource + - test_run_script + - deploy_workspace_item + toolCallArgs: + - tool: get_db_schema + field: resourcePath + stringIncludesAnyOf: + - f/data/reports_pg + skipJudge: true + judgeChecklist: + - fetches the schema through get_db_schema with the resource path f/data/reports_pg + - when the lookup fails, tells the user instead of inventing table names + - does not write scripts or resources to answer a read-only question + +# --- API catalog (search_api_endpoints / call_api_get / call_api_endpoint) --- +# The harness serves the catalog and the executed calls itself (mock +# listMcpTools + benchmark fetch handlers in adapters/frontend), so these cases +# do not require an mcp-enabled eval backend. + +- id: global-test30-api-catalog-workers + prompt: |- + Are any workers online right now? Give me a quick summary of the workers and when they last pinged. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_empty.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - search_api_endpoints + - call_api_get + forbiddenToolsUsed: + - call_api_endpoint + - write_script + - deploy_workspace_item + toolCallArgs: + - tool: call_api_get + field: name + stringIncludesAnyOf: + - listWorkers + # Read-only workspace inspection produces no draft; validate via tool use. + skipJudge: true + judgeChecklist: + - discovers the workers endpoint through the API catalog instead of guessing or fabricating + - reports worker status from the returned data + +- id: global-test31-draft-test-run-not-deployed + prompt: |- + Update `f/evals/global/format_greeting` so the provided name is uppercased in the greeting, then run it with name "ada" to check it works. + Keep it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/format_greeting + language: bun + valueIncludes: + - toUpperCase + toolExpect: + requiredToolsUsed: + - test_run_script + forbiddenToolsUsed: + - call_api_endpoint + - deploy_workspace_item + - delete_workspace_item + # The judge only sees the drafts artifact and cannot observe runs, so it always + # docks the prompt's "run it" requirement — validate deterministically instead: + # draft content via valueIncludes, the test run via toolExpect (test_run_script + # required, call_api_endpoint forbidden). + skipJudge: true + judgeChecklist: + - creates an AI draft of f/evals/global/format_greeting with the name uppercased in the greeting + - does not deploy or save the draft + +- id: global-test32-delete-via-dedicated-tool + prompt: |- + Delete the deployed script `f/evals/global/send_report_digest` from the workspace — we don't use it anymore. + initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - delete_workspace_item + forbiddenToolsUsed: + - call_api_endpoint + - search_api_endpoints + - write_script + # Deletion produces no draft; validate via tool use. + skipJudge: true + judgeChecklist: + - deletes the deployed script via delete_workspace_item rather than a raw API endpoint + +- id: global-draft-diff-report + prompt: |- + Update the existing workspace script at `f/evals/global/format_greeting` so the returned message ends with an exclamation mark, keeping everything else the same. + Then show me exactly what your draft changes compared to the deployed version. + Leave the result as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/format_greeting + language: bun + valueIncludes: + - "!" + toolExpect: + requiredToolsUsed: + - diff + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: diff + field: path + stringIncludesAnyOf: + - f/evals/global/format_greeting + judgeChecklist: + - creates an AI draft for the existing f/evals/global/format_greeting script with the exclamation-mark change + - the draft changes only the returned message's punctuation — summary, language, path, and the rest of the code are untouched + - does not deploy or save the draft to the workspace + +- id: global-resource-manual-credentials + prompt: |- + Set up a resource for our production Postgres database at `f/evals/global/prod_db` (host db.internal.example.com, port 5432, database `orders`, user `app`). + I don't want to paste the password into this chat — prepare everything so I can enter it myself. + Leave it as an AI draft only; do not deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 12 + validate: + draftCountAtLeast: 1 + requiredDrafts: + - type: resource + path: f/evals/global/prod_db + valueIncludes: + - db.internal.example.com + - orders + toolExpect: + requiredToolsUsed: + - write_resource + - open_page + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + # The model may land the user in the resource's drawer or in the drawer of + # the secret variable it created for the password — both are correct. + - tool: open_page + field: page + stringIncludesAnyOf: + - resources + - variables + - tool: open_page + field: open + stringIncludesAnyOf: + - prod_db + - password + judgeChecklist: + - creates a postgres resource draft at f/evals/global/prod_db with the provided host, port, database, and user + - the password is left for the user to provide (empty, a placeholder, or a secret variable reference) — no invented password value presented as real + - does not deploy or save anything to the workspace + +- id: global-test29-email-trigger-draft + prompt: |- + Set up a draft auto-reply job. + Create a Bun script at `f/evals/global/email_pong` that returns the string "pong". + Then set it up so it runs whenever an email is received at the inbox `pong`. + Leave everything as AI drafts only; do not deploy or save anything to the workspace. + runtime: + maxTurns: 10 + validate: + requiredDrafts: + - type: script + path: f/evals/global/email_pong + language: bun + valueIncludes: + - pong + toolExpect: + requiredToolsUsed: + - write_script + - write_trigger + forbiddenToolsUsed: + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + # "Runs when an email is received" must resolve to the native email trigger kind, + # never a faked HTTP webhook. Assert the recorded tool-call kind (not the draft): + # it holds even on a CE backend where email trigger routes (smtp+private) 404. + - tool: write_trigger + field: kind + stringIncludesAnyOf: + - email + skipJudge: true diff --git a/ai_evals/core/types.ts b/ai_evals/core/types.ts index f142bc8d36..83e9a218f3 100644 --- a/ai_evals/core/types.ts +++ b/ai_evals/core/types.ts @@ -31,6 +31,8 @@ export interface EvalCaseRuntimeSpec { maxTurns?: number; backendPreview?: EvalCaseRuntimeBackendPreview; appContext?: EvalCaseRuntimeAppContextSpec; + // Global mode: run as a session chat (preview tools + session prompt) vs the standalone chat. + sessionChat?: boolean; } export interface FlowValidationSpec { diff --git a/ai_evals/core/validators.test.ts b/ai_evals/core/validators.test.ts index 5183a20226..d603dae626 100644 --- a/ai_evals/core/validators.test.ts +++ b/ai_evals/core/validators.test.ts @@ -174,6 +174,40 @@ describe("validateToolExpectations", () => { expect(checks.every((check) => check.passed)).toBe(true); }); + it("accepts a stringIncludesAnyOf substring inside an array-valued field", () => { + const checks = validateToolExpectations({ + run: { + success: true, + actual: {}, + assistantMessageCount: 1, + toolCallCount: 1, + toolsUsed: ["open_page"], + toolCallDetails: [ + { + name: "open_page", + arguments: { + page: "compare", + items: ["script:f/evals/global/compare_review_demo"], + }, + }, + ], + skillsInvoked: [], + }, + toolExpect: { + requiredToolsUsed: ["open_page"], + toolCallArgs: [ + { + tool: "open_page", + field: "items", + stringIncludesAnyOf: ["f/evals/global/compare_review_demo"], + }, + ], + }, + }); + + expect(checks.every((check) => check.passed)).toBe(true); + }); + it("accepts stringIncludesAnyOf when only one of several calls matches", () => { // Existential: a mutation mixed with verification SELECTs still passes. const checks = validateToolExpectations({ diff --git a/ai_evals/core/validators.ts b/ai_evals/core/validators.ts index 7570ccd12d..32ff225fc1 100644 --- a/ai_evals/core/validators.ts +++ b/ai_evals/core/validators.ts @@ -239,9 +239,15 @@ export function validateToolExpectations(input: { // model mixes the requested statement (e.g. an UPDATE) with verification // SELECTs that would otherwise fail an "all calls" check. const needles = rule.stringIncludesAnyOf.map((needle) => needle.toLowerCase()); - const hasMatch = values.some( - (value) => - typeof value === "string" && needles.some((needle) => value.toLowerCase().includes(needle)) + // Array-valued fields (e.g. open_page.items) match on any element. + const haystacks = (value: unknown): string[] => + typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.filter((v): v is string => typeof v === "string") + : []; + const hasMatch = values.some((value) => + haystacks(value).some((hay) => needles.some((needle) => hay.toLowerCase().includes(needle))) ); checks.push( check( diff --git a/ai_evals/modes/global.ts b/ai_evals/modes/global.ts index 050a4caad9..5628c21d5d 100644 --- a/ai_evals/modes/global.ts +++ b/ai_evals/modes/global.ts @@ -41,6 +41,7 @@ export function createGlobalModeRunner( workspaceFixtures: initial?.workspace, liveEditorDrafts: initial?.liveEditorDrafts, user: initial?.user, + sessionChat: context.evalCase?.runtime?.sessionChat, maxIterations: context.evalCase?.runtime?.maxTurns, provider: modelConfig.provider, model: modelConfig.model, diff --git a/backend/.sqlx/query-020cc73e39782e7be50a146373096bd4378988af6447feeafd243262c246a9f5.json b/backend/.sqlx/query-020cc73e39782e7be50a146373096bd4378988af6447feeafd243262c246a9f5.json new file mode 100644 index 0000000000..6ab08ab29a --- /dev/null +++ b/backend/.sqlx/query-020cc73e39782e7be50a146373096bd4378988af6447feeafd243262c246a9f5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args->'__git_sync_auto_pull' FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "020cc73e39782e7be50a146373096bd4378988af6447feeafd243262c246a9f5" +} diff --git a/backend/.sqlx/query-02e39bb9957d3ccfe3d573994cd281a22467010d2c4a0611c8ae02742336f433.json b/backend/.sqlx/query-02e39bb9957d3ccfe3d573994cd281a22467010d2c4a0611c8ae02742336f433.json new file mode 100644 index 0000000000..9219641ce9 --- /dev/null +++ b/backend/.sqlx/query-02e39bb9957d3ccfe3d573994cd281a22467010d2c4a0611c8ae02742336f433.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO amqp_trigger (\n amqp_resource_path,\n queue_name,\n exchange,\n options,\n workspace_id,\n path,\n script_path,\n is_flow,\n permissioned_as,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "02e39bb9957d3ccfe3d573994cd281a22467010d2c4a0611c8ae02742336f433" +} diff --git a/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json b/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json new file mode 100644 index 0000000000..bacf0f76c2 --- /dev/null +++ b/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de" +} diff --git a/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json b/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json new file mode 100644 index 0000000000..9574582d5f --- /dev/null +++ b/backend/.sqlx/query-042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "042c42957221352891f7433f2ec0d6b7f686d9c7d505590eec7ba3bfeeb406ca" +} diff --git a/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json b/backend/.sqlx/query-042ff3003bf82d11a78a1074a63379c19880e4d66dbf4dd5391f053b6dfa6b01.json similarity index 57% rename from backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json rename to backend/.sqlx/query-042ff3003bf82d11a78a1074a63379c19880e4d66dbf4dd5391f053b6dfa6b01.json index e810fc4754..fef2943c22 100644 --- a/backend/.sqlx/query-36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c.json +++ b/backend/.sqlx/query-042ff3003bf82d11a78a1074a63379c19880e4d66dbf4dd5391f053b6dfa6b01.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"", + "query": "INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3::text::jsonb, $10, $5, $6,\n flow_status, workflow_as_code_status,\n $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END AS status,\n q.worker\n FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb RETURNING duration_ms AS \"duration_ms!\"", "describe": { "columns": [ { @@ -13,7 +13,7 @@ "Left": [ "Uuid", "Bool", - "Jsonb", + "Text", "Bool", "Varchar", "Text", @@ -27,5 +27,5 @@ false ] }, - "hash": "36c4e57afcab22f4b6825ccebe47767b8a8fe0a638250f7c7777e5a9f7530e5c" + "hash": "042ff3003bf82d11a78a1074a63379c19880e4d66dbf4dd5391f053b6dfa6b01" } diff --git a/backend/.sqlx/query-05843e117361d6b7cd9da652596d3be7a970d602d3809396d9253d7e5dc88152.json b/backend/.sqlx/query-05843e117361d6b7cd9da652596d3be7a970d602d3809396d9253d7e5dc88152.json new file mode 100644 index 0000000000..950fe7bcdf --- /dev/null +++ b/backend/.sqlx/query-05843e117361d6b7cd9da652596d3be7a970d602d3809396d9253d7e5dc88152.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT kind, path, script_path, is_flow FROM (\n SELECT 'schedule' AS kind, path, script_path, is_flow FROM schedule\n WHERE workspace_id = $1\n AND script_path IS NOT NULL\n UNION ALL\n SELECT 'email', path, script_path, is_flow FROM email_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'kafka', path, script_path, is_flow FROM kafka_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'amqp', path, script_path, is_flow FROM amqp_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'nats', path, script_path, is_flow FROM nats_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'postgres', path, script_path, is_flow FROM postgres_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'sqs', path, script_path, is_flow FROM sqs_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'gcp', path, script_path, is_flow FROM gcp_trigger\n WHERE workspace_id = $1\n ) t\n WHERE ($2::text IS NULL OR script_path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "05843e117361d6b7cd9da652596d3be7a970d602d3809396d9253d7e5dc88152" +} diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index d29a18c691..e7ed0aee65 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, true, true ] diff --git a/backend/.sqlx/query-079e1dab42a783a1e5e4cba5faa854b4b727b1fab89e42f2efaf97c63dad6384.json b/backend/.sqlx/query-079e1dab42a783a1e5e4cba5faa854b4b727b1fab89e42f2efaf97c63dad6384.json new file mode 100644 index 0000000000..8c58b6cf54 --- /dev/null +++ b/backend/.sqlx/query-079e1dab42a783a1e5e4cba5faa854b4b727b1fab89e42f2efaf97c63dad6384.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "079e1dab42a783a1e5e4cba5faa854b4b727b1fab89e42f2efaf97c63dad6384" +} diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index 6efb66005d..a762af65c9 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -36,7 +36,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-0bb3de62f920a4f6c5da6f0a7bfc780a5ac35fa7639c4ee3a449b693add24241.json b/backend/.sqlx/query-0bb3de62f920a4f6c5da6f0a7bfc780a5ac35fa7639c4ee3a449b693add24241.json new file mode 100644 index 0000000000..1451621b1e --- /dev/null +++ b/backend/.sqlx/query-0bb3de62f920a4f6c5da6f0a7bfc780a5ac35fa7639c4ee3a449b693add24241.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n (elem->>'installation_id')::bigint as installation_id,\n elem->>'github_base_url' as github_base_url\n FROM workspace_settings,\n LATERAL jsonb_array_elements(git_app_installations) AS elem\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "installation_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "github_base_url", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "0bb3de62f920a4f6c5da6f0a7bfc780a5ac35fa7639c4ee3a449b693add24241" +} diff --git a/backend/.sqlx/query-0bd853be57b43a7820cc1d73d92b8babd50b1092d4ad70e022431bdcf32e378f.json b/backend/.sqlx/query-0bd853be57b43a7820cc1d73d92b8babd50b1092d4ad70e022431bdcf32e378f.json new file mode 100644 index 0000000000..f0e1c853d8 --- /dev/null +++ b/backend/.sqlx/query-0bd853be57b43a7820cc1d73d92b8babd50b1092d4ad70e022431bdcf32e378f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, 'test-workspace', 'other')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0bd853be57b43a7820cc1d73d92b8babd50b1092d4ad70e022431bdcf32e378f" +} diff --git a/backend/.sqlx/query-0e016aaa6842767ac636e5b3e1febfa9cee08b93e0562beb629d84c4c689f4e7.json b/backend/.sqlx/query-0e016aaa6842767ac636e5b3e1febfa9cee08b93e0562beb629d84c4c689f4e7.json new file mode 100644 index 0000000000..6f5d84d2d6 --- /dev/null +++ b/backend/.sqlx/query-0e016aaa6842767ac636e5b3e1febfa9cee08b93e0562beb629d84c4c689f4e7.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE descendants AS (\n SELECT id, 0 AS depth FROM workspace\n WHERE parent_workspace_id = $1 AND NOT deleted\n UNION ALL\n SELECT w.id, d.depth + 1 FROM workspace w\n JOIN descendants d ON w.parent_workspace_id = d.id\n WHERE NOT w.deleted AND d.depth < 10\n )\n SELECT id as \"id!\" FROM descendants WHERE (id = $2 OR id = $3)\n ORDER BY (id = $2) DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0e016aaa6842767ac636e5b3e1febfa9cee08b93e0562beb629d84c4c689f4e7" +} diff --git a/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json b/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json index 0e0fbca58e..9333553d8e 100644 --- a/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json +++ b/backend/.sqlx/query-0f7e01b613a94b29784aae6d7b17b23d6dcf2e5364852a5e85b3c41c417bace2.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json b/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json index 0e966467a9..9efcf8fc44 100644 --- a/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json +++ b/backend/.sqlx/query-16d438374b03a9c515f4c2d638366f38ffe2f3a0958adea53e67757c6ac463ec.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json new file mode 100644 index 0000000000..bc2e6a6f63 --- /dev/null +++ b/backend/.sqlx/query-17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(username, split_part(email, '@', 1)) AS \"username!\", email FROM password WHERE super_admin = true AND disabled = false ORDER BY email LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + false + ] + }, + "hash": "17cdf02b4912078459526205849246fd2bdf9e3fce89852120aa4ad4ad6abfc3" +} diff --git a/backend/.sqlx/query-1b32339718406531acd2d051386d2817e9ba0b3a66f7eb4fc1177764563a0665.json b/backend/.sqlx/query-1b32339718406531acd2d051386d2817e9ba0b3a66f7eb4fc1177764563a0665.json new file mode 100644 index 0000000000..a409f83519 --- /dev/null +++ b/backend/.sqlx/query-1b32339718406531acd2d051386d2817e9ba0b3a66f7eb4fc1177764563a0665.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND NOT deleted AND is_dev_workspace AND COALESCE(dev_workspace_label, 'dev') = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1b32339718406531acd2d051386d2817e9ba0b3a66f7eb4fc1177764563a0665" +} diff --git a/backend/.sqlx/query-1bf87fe9667f860c6089bffb803291f4ff5c979be956369fa4406d789cca92fc.json b/backend/.sqlx/query-1bf87fe9667f860c6089bffb803291f4ff5c979be956369fa4406d789cca92fc.json new file mode 100644 index 0000000000..1ee774a993 --- /dev/null +++ b/backend/.sqlx/query-1bf87fe9667f860c6089bffb803291f4ff5c979be956369fa4406d789cca92fc.json @@ -0,0 +1,112 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n j.id AS \"id!\", j.workspace_id AS \"workspace_id!\", j.parent_job, j.flow_step_id IS NOT NULL AS \"is_flow_step?\",\n COALESCE(s.flow_status, s.workflow_as_code_status)::text AS \"flow_status: Box\", r.ping AS last_ping, j.same_worker AS \"same_worker?\",\n q.worker AS \"worker?\",\n wp.ping_at AS \"worker_last_ping?\",\n wp.memory_usage AS \"worker_memory_usage?\",\n wp.wm_memory_usage AS \"worker_wm_memory_usage?\",\n wp.memory AS \"worker_memory_total?\",\n wp.worker_group AS \"worker_group?\",\n wp.wm_version AS \"worker_version?\",\n wp.current_job_id AS \"worker_current_job_id?\",\n wp.worker_instance AS \"worker_instance?\"\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n LEFT JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null AND q.scheduled_for <= now()\n AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode' OR j.kind = 'singlestepflow')\n AND r.ping IS NOT NULL AND r.ping < NOW() - ($1 || ' seconds')::interval\n AND q.canceled_by IS NULL\n\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "is_flow_step?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "flow_status: Box", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "last_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "same_worker?", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "worker?", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "worker_last_ping?", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "worker_memory_usage?", + "type_info": "Int8" + }, + { + "ordinal": 10, + "name": "worker_wm_memory_usage?", + "type_info": "Int8" + }, + { + "ordinal": 11, + "name": "worker_memory_total?", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "worker_group?", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "worker_version?", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "worker_current_job_id?", + "type_info": "Uuid" + }, + { + "ordinal": 15, + "name": "worker_instance?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + null, + null, + true, + false, + true, + false, + true, + true, + true, + false, + false, + true, + false + ] + }, + "hash": "1bf87fe9667f860c6089bffb803291f4ff5c979be956369fa4406d789cca92fc" +} diff --git a/backend/.sqlx/query-1c018b483795d9d33baa7be2ee41f48da6fe47bc29b6987cf60edd56bb1663b6.json b/backend/.sqlx/query-1c018b483795d9d33baa7be2ee41f48da6fe47bc29b6987cf60edd56bb1663b6.json new file mode 100644 index 0000000000..6449a361e6 --- /dev/null +++ b/backend/.sqlx/query-1c018b483795d9d33baa7be2ee41f48da6fe47bc29b6987cf60edd56bb1663b6.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT w.parent_workspace_id, w.is_dev_workspace, w.dev_workspace_label,\n p.is_dev_workspace as \"parent_is_dev_workspace?\", p.dev_workspace_label as \"parent_dev_label?\"\n FROM workspace w LEFT JOIN workspace p ON p.id = w.parent_workspace_id\n WHERE w.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "dev_workspace_label", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "parent_is_dev_workspace?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "parent_dev_label?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + true + ] + }, + "hash": "1c018b483795d9d33baa7be2ee41f48da6fe47bc29b6987cf60edd56bb1663b6" +} diff --git a/backend/.sqlx/query-1cd16f93bdc90b24ceb1e86495069c3592218d34e58d617253e8b563bc661a7a.json b/backend/.sqlx/query-1cd16f93bdc90b24ceb1e86495069c3592218d34e58d617253e8b563bc661a7a.json new file mode 100644 index 0000000000..f2327d1700 --- /dev/null +++ b/backend/.sqlx/query-1cd16f93bdc90b24ceb1e86495069c3592218d34e58d617253e8b563bc661a7a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH t1 AS (UPDATE websocket_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t2 AS (UPDATE kafka_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t3 AS (UPDATE postgres_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t4 AS (UPDATE mqtt_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t5 AS (UPDATE nats_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t6 AS (UPDATE sqs_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), t7 AS (UPDATE amqp_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4) UPDATE gcp_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "1cd16f93bdc90b24ceb1e86495069c3592218d34e58d617253e8b563bc661a7a" +} diff --git a/backend/.sqlx/query-1ce0728a6b0942fecf8ce5f4a06857a2601b10b057877c3b2795b8ea9c928c1b.json b/backend/.sqlx/query-1ce0728a6b0942fecf8ce5f4a06857a2601b10b057877c3b2795b8ea9c928c1b.json new file mode 100644 index 0000000000..7a74d18f48 --- /dev/null +++ b/backend/.sqlx/query-1ce0728a6b0942fecf8ce5f4a06857a2601b10b057877c3b2795b8ea9c928c1b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running)\n VALUES ($1, $2, now() + ($3::bigint::text || ' s')::interval, 'other', $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Int8", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "1ce0728a6b0942fecf8ce5f4a06857a2601b10b057877c3b2795b8ea9c928c1b" +} diff --git a/backend/.sqlx/query-20ae664da5ebd70a6d37609f6b9a3e10d8d524fbaaa8eddf74f12a906b3c8306.json b/backend/.sqlx/query-20ae664da5ebd70a6d37609f6b9a3e10d8d524fbaaa8eddf74f12a906b3c8306.json new file mode 100644 index 0000000000..04e663f7c3 --- /dev/null +++ b/backend/.sqlx/query-20ae664da5ebd70a6d37609f6b9a3e10d8d524fbaaa8eddf74f12a906b3c8306.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running)\n VALUES ($1, 'test-workspace', now() + ($2::bigint::text || ' s')::interval, 'other', $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int8", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "20ae664da5ebd70a6d37609f6b9a3e10d8d524fbaaa8eddf74f12a906b3c8306" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index 704883d4f1..831dce83d7 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -40,7 +40,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } @@ -79,7 +80,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json b/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json index 49a2977cf8..54f9e57c2f 100644 --- a/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json +++ b/backend/.sqlx/query-22e0e8a1aa48f8b21763452bd36fbe7db4887c4ac5295052c796bd78a7edc50b.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json b/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json index 5c84181bdb..6aa714182f 100644 --- a/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json +++ b/backend/.sqlx/query-23419adcd74c326d716527293eff518b42f4cdb33e034441015494bd26c172d2.json @@ -43,7 +43,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json b/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json new file mode 100644 index 0000000000..f9cf82a9fd --- /dev/null +++ b/backend/.sqlx/query-2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.workspace_id, s.path\n FROM schedule s JOIN workspace w ON w.id = s.workspace_id AND NOT w.deleted\n WHERE s.enabled IS TRUE\n AND NOT EXISTS (\n SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id)\n WHERE j.workspace_id = s.workspace_id\n AND j.trigger_kind = 'schedule'\n AND j.trigger = s.path\n AND j.runnable_path = s.script_path\n AND j.parent_job IS NULL\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "2756fab586489be33a3031fabfdc87be642091a2f8a6973d001e59be892a984d" +} diff --git a/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json b/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json index 26a8ba2ee2..03e920f705 100644 --- a/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json +++ b/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json @@ -5,7 +5,7 @@ "columns": [], "parameters": { "Left": [ - "Varchar" + "Text" ] }, "nullable": [] diff --git a/backend/.sqlx/query-2bed492ef32edf36e60e8a03268fa25bfb67dd641153d1ca23f7d0d2ae73624e.json b/backend/.sqlx/query-2bed492ef32edf36e60e8a03268fa25bfb67dd641153d1ca23f7d0d2ae73624e.json index 241778d359..920a463d01 100644 --- a/backend/.sqlx/query-2bed492ef32edf36e60e8a03268fa25bfb67dd641153d1ca23f7d0d2ae73624e.json +++ b/backend/.sqlx/query-2bed492ef32edf36e60e8a03268fa25bfb67dd641153d1ca23f7d0d2ae73624e.json @@ -43,7 +43,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json b/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json new file mode 100644 index 0000000000..2a67e64615 --- /dev/null +++ b/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO feature_usage (feature, kind, key, entity_id, value)\n SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[])\n ON CONFLICT (feature, kind, key, entity_id, day)\n DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3" +} diff --git a/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json b/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json new file mode 100644 index 0000000000..8714711595 --- /dev/null +++ b/backend/.sqlx/query-2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2dc5a74c4e614b43148925cdbac234f6a2f295a9c46897f5e9aab8f7298ff0d7" +} diff --git a/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json b/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json new file mode 100644 index 0000000000..0a833ba620 --- /dev/null +++ b/backend/.sqlx/query-3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username, email FROM usr WHERE workspace_id = $1 AND is_admin = true AND operator = false AND disabled = false ORDER BY username LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "3202bed875693ae923f496272cd8ad89b2f17a9d3ef4659c2d2284415177b32c" +} diff --git a/backend/.sqlx/query-3451cbd6a783da68324f64c2e95ad146e2ca8c1cd7c10891fe21a7638aedc381.json b/backend/.sqlx/query-3451cbd6a783da68324f64c2e95ad146e2ca8c1cd7c10891fe21a7638aedc381.json new file mode 100644 index 0000000000..a5275d2ed1 --- /dev/null +++ b/backend/.sqlx/query-3451cbd6a783da68324f64c2e95ad146e2ca8c1cd7c10891fe21a7638aedc381.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE up AS (\n SELECT id, parent_workspace_id, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, up.depth + 1\n FROM workspace w JOIN up ON w.id = up.parent_workspace_id\n WHERE up.depth < 10\n )\n SELECT git_sync FROM workspace_settings\n WHERE workspace_id = (SELECT id FROM up WHERE parent_workspace_id IS NULL LIMIT 1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "3451cbd6a783da68324f64c2e95ad146e2ca8c1cd7c10891fe21a7638aedc381" +} diff --git a/backend/.sqlx/query-387e135bf363533089493c35eb39dd4864f4e0aa0093df47df7d444ee748dbb2.json b/backend/.sqlx/query-387e135bf363533089493c35eb39dd4864f4e0aa0093df47df7d444ee748dbb2.json new file mode 100644 index 0000000000..50f0bf86e5 --- /dev/null +++ b/backend/.sqlx/query-387e135bf363533089493c35eb39dd4864f4e0aa0093df47df7d444ee748dbb2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM v2_job_completed\n WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "387e135bf363533089493c35eb39dd4864f4e0aa0093df47df7d444ee748dbb2" +} diff --git a/backend/.sqlx/query-392681cc6aa1ed2119d33aa68c8de907889fdfab77fa3af85a96ddaa068af733.json b/backend/.sqlx/query-392681cc6aa1ed2119d33aa68c8de907889fdfab77fa3af85a96ddaa068af733.json new file mode 100644 index 0000000000..247a7d23f3 --- /dev/null +++ b/backend/.sqlx/query-392681cc6aa1ed2119d33aa68c8de907889fdfab77fa3af85a96ddaa068af733.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE schedule SET error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled IS TRUE", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "392681cc6aa1ed2119d33aa68c8de907889fdfab77fa3af85a96ddaa068af733" +} diff --git a/backend/.sqlx/query-3b7498768197d7e002c3965ad16e6b88daa38ba272d22f846bd4267a134c404c.json b/backend/.sqlx/query-3b7498768197d7e002c3965ad16e6b88daa38ba272d22f846bd4267a134c404c.json new file mode 100644 index 0000000000..266c75fad3 --- /dev/null +++ b/backend/.sqlx/query-3b7498768197d7e002c3965ad16e6b88daa38ba272d22f846bd4267a134c404c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(\n git_sync,\n '{repositories}',\n (SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' = $2\n AND jsonb_typeof(elem->'auto_pull') = 'object'\n THEN jsonb_set(elem, '{auto_pull}',\n ((elem->'auto_pull') - 'webhook_id' - 'webhook_secret' - 'webhook_error') || $3)\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "3b7498768197d7e002c3965ad16e6b88daa38ba272d22f846bd4267a134c404c" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index f5ee767768..257e1e528f 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -129,7 +129,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json b/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json deleted file mode 100644 index 3d91feace1..0000000000 --- a/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT provider, model, mode,\n COUNT(*)::BIGINT as \"session_count!\",\n COALESCE(SUM(message_count), 0)::BIGINT as \"message_count!\"\n FROM ai_chat_usage\n WHERE created_at > NOW() - INTERVAL '30 days'\n GROUP BY provider, model, mode\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "provider", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "model", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "mode", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "session_count!", - "type_info": "Int8" - }, - { - "ordinal": 4, - "name": "message_count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - null, - null - ] - }, - "hash": "3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943" -} diff --git a/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json b/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json new file mode 100644 index 0000000000..114712fc28 --- /dev/null +++ b/backend/.sqlx/query-40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "40c591ab93adf3bb4f17598f78b76705eeda7d17d49cafb3e9e6285dee6804c9" +} diff --git a/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json b/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json index ad5613b8fd..a2174020cd 100644 --- a/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json +++ b/backend/.sqlx/query-42b4b73e9d60348e2d90fcade9dcad6d8995242dc20a4e14c1a8fae4fc6a9fd2.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-4371e59b85cc279dba03800aacde8a4b96c9629cdc785edc7a389c3eb7269608.json b/backend/.sqlx/query-4371e59b85cc279dba03800aacde8a4b96c9629cdc785edc7a389c3eb7269608.json new file mode 100644 index 0000000000..4bf6f3692b --- /dev/null +++ b/backend/.sqlx/query-4371e59b85cc279dba03800aacde8a4b96c9629cdc785edc7a389c3eb7269608.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (kind = 'flow' AND parent_job IS NULL) AS \"restartable!\"\n FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "restartable!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4371e59b85cc279dba03800aacde8a4b96c9629cdc785edc7a389c3eb7269608" +} diff --git a/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json b/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json index e61104d62d..a5db66e7c6 100644 --- a/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json +++ b/backend/.sqlx/query-45024b932383199974616bba1fc2f7175cc6f2e02d9c565bb5159cae3e0b6835.json @@ -33,7 +33,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json b/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json index c269f7f340..dcd3573e92 100644 --- a/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json +++ b/backend/.sqlx/query-46f00a75b2e7e4ac70758a9687070f68bc0421f1aa228f80157adda63191d33b.json @@ -42,7 +42,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-4a35f9a0201e283d854bd77f7f79d4c24699c2c7abad4086e0bb876c5c6b2c38.json b/backend/.sqlx/query-4a35f9a0201e283d854bd77f7f79d4c24699c2c7abad4086e0bb876c5c6b2c38.json new file mode 100644 index 0000000000..6c5b364a5b --- /dev/null +++ b/backend/.sqlx/query-4a35f9a0201e283d854bd77f7f79d4c24699c2c7abad4086e0bb876c5c6b2c38.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor',\n flow_status = $2 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "4a35f9a0201e283d854bd77f7f79d4c24699c2c7abad4086e0bb876c5c6b2c38" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 7474818bf3..8849c7f0fa 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -81,7 +81,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json b/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json index 99d6e818ea..6425204e0e 100644 --- a/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json +++ b/backend/.sqlx/query-4d272cf4a77aab7007a5b35589e08532a1020cabaf5e22325a1e05f0491d785c.json @@ -40,7 +40,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json index bdb39d134f..3a9c2ffae7 100644 --- a/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json +++ b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json @@ -47,7 +47,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json index dea8f68e4f..ee1034e845 100644 --- a/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json +++ b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json @@ -35,7 +35,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } @@ -76,7 +77,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-521367aaea7beefe4ff9cdb66273f8e3cddfbaf598536f1753f0824f84604826.json b/backend/.sqlx/query-521367aaea7beefe4ff9cdb66273f8e3cddfbaf598536f1753f0824f84604826.json new file mode 100644 index 0000000000..9acef34c47 --- /dev/null +++ b/backend/.sqlx/query-521367aaea7beefe4ff9cdb66273f8e3cddfbaf598536f1753f0824f84604826.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_queue", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "521367aaea7beefe4ff9cdb66273f8e3cddfbaf598536f1753f0824f84604826" +} diff --git a/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json b/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json index d81d6ebd94..6513bbbf21 100644 --- a/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json +++ b/backend/.sqlx/query-54ad5cc89563fdbbacdd6bfde9be6ecfcdec3d505d28cc65b5920dcf87c0014f.json @@ -36,7 +36,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-560539adbad0ecfa57fa477c3b82d82c350857166fc27fe9eecc88bcc4b229bc.json b/backend/.sqlx/query-560539adbad0ecfa57fa477c3b82d82c350857166fc27fe9eecc88bcc4b229bc.json index 91c941593f..81ca75de73 100644 --- a/backend/.sqlx/query-560539adbad0ecfa57fa477c3b82d82c350857166fc27fe9eecc88bcc4b229bc.json +++ b/backend/.sqlx/query-560539adbad0ecfa57fa477c3b82d82c350857166fc27fe9eecc88bcc4b229bc.json @@ -43,7 +43,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-5884ce1906015f6b96231f311226e490d5dfcdd7a94dcbe5d05c9e5af37ac4a4.json b/backend/.sqlx/query-5884ce1906015f6b96231f311226e490d5dfcdd7a94dcbe5d05c9e5af37ac4a4.json index 0c445c1ae5..9a18371f1f 100644 --- a/backend/.sqlx/query-5884ce1906015f6b96231f311226e490d5dfcdd7a94dcbe5d05c9e5af37ac4a4.json +++ b/backend/.sqlx/query-5884ce1906015f6b96231f311226e490d5dfcdd7a94dcbe5d05c9e5af37ac4a4.json @@ -36,7 +36,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-58a79f32f880c5ea0ee8821ca4ad07f0f0519a31300e597776ad8b77130501d2.json b/backend/.sqlx/query-58a79f32f880c5ea0ee8821ca4ad07f0f0519a31300e597776ad8b77130501d2.json new file mode 100644 index 0000000000..27596043f8 --- /dev/null +++ b/backend/.sqlx/query-58a79f32f880c5ea0ee8821ca4ad07f0f0519a31300e597776ad8b77130501d2.json @@ -0,0 +1,63 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT script_path, table_path, kind, name, expr, filter FROM data_metric dm WHERE dm.workspace_id = $1 AND ($2::text IS NULL OR dm.table_path = $2) AND ($3::text IS NULL OR dm.script_path = $3 OR dm.script_path LIKE $4) AND ($6::text IS NULL OR (dm.table_path, dm.kind, dm.name, dm.script_path) > ($6, $7, $8, $9)) AND ( $10 OR dm.script_path = ANY($11) OR EXISTS ( SELECT 1 FROM unnest($12::text[]) AS pfx WHERE dm.script_path = pfx OR left(dm.script_path, length(pfx) + 1) = pfx || '/' ) ) AND EXISTS ( SELECT 1 FROM script s WHERE s.workspace_id = dm.workspace_id AND s.path = dm.script_path AND s.archived = false AND s.deleted = false ) ORDER BY dm.table_path, dm.kind, dm.name, dm.script_path LIMIT $5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "table_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "expr", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "filter", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Int8", + "Text", + "Text", + "Text", + "Text", + "Bool", + "TextArray", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + true + ] + }, + "hash": "58a79f32f880c5ea0ee8821ca4ad07f0f0519a31300e597776ad8b77130501d2" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json b/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json new file mode 100644 index 0000000000..353920fdeb --- /dev/null +++ b/backend/.sqlx/query-5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf.json @@ -0,0 +1,208 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n slack_team_id,\n teams_team_id,\n teams_team_name,\n teams_team_guid,\n slack_name,\n slack_command_script,\n teams_command_script,\n slack_email,\n slack_oauth_client_id,\n slack_oauth_client_secret,\n customer_id,\n plan,\n webhook,\n deploy_to,\n ai_config,\n large_file_storage,\n datatable,\n ducklake,\n git_sync,\n deploy_ui,\n default_app,\n default_scripts,\n mute_critical_alerts,\n color,\n operator_settings,\n git_app_installations,\n auto_invite,\n error_handler,\n success_handler,\n public_app_execution_limit_per_minute,\n error_handler_fallback_to_instance_alerts\n FROM\n workspace_settings\n WHERE\n workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "slack_team_id", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "teams_team_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "teams_team_name", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "teams_team_guid", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "slack_name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "slack_command_script", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "teams_command_script", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "slack_email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "slack_oauth_client_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "slack_oauth_client_secret", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 12, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "webhook", + "type_info": "Text" + }, + { + "ordinal": 14, + "name": "deploy_to", + "type_info": "Varchar" + }, + { + "ordinal": 15, + "name": "ai_config", + "type_info": "Jsonb" + }, + { + "ordinal": 16, + "name": "large_file_storage", + "type_info": "Jsonb" + }, + { + "ordinal": 17, + "name": "datatable", + "type_info": "Jsonb" + }, + { + "ordinal": 18, + "name": "ducklake", + "type_info": "Jsonb" + }, + { + "ordinal": 19, + "name": "git_sync", + "type_info": "Jsonb" + }, + { + "ordinal": 20, + "name": "deploy_ui", + "type_info": "Jsonb" + }, + { + "ordinal": 21, + "name": "default_app", + "type_info": "Varchar" + }, + { + "ordinal": 22, + "name": "default_scripts", + "type_info": "Jsonb" + }, + { + "ordinal": 23, + "name": "mute_critical_alerts", + "type_info": "Bool" + }, + { + "ordinal": 24, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 26, + "name": "git_app_installations", + "type_info": "Jsonb" + }, + { + "ordinal": 27, + "name": "auto_invite", + "type_info": "Jsonb" + }, + { + "ordinal": 28, + "name": "error_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 29, + "name": "success_handler", + "type_info": "Jsonb" + }, + { + "ordinal": 30, + "name": "public_app_execution_limit_per_minute", + "type_info": "Int4" + }, + { + "ordinal": 31, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "5ae9ad14effe923f1952d0e1d1f67ad00ff6c13918d46dd474cc48824b12ccaf" +} diff --git a/backend/.sqlx/query-5f2e66e53166840c68efbc521fe9393023e79f948976e6ffc0e9bfc94623393c.json b/backend/.sqlx/query-5f2e66e53166840c68efbc521fe9393023e79f948976e6ffc0e9bfc94623393c.json new file mode 100644 index 0000000000..2a9c1c119e --- /dev/null +++ b/backend/.sqlx/query-5f2e66e53166840c68efbc521fe9393023e79f948976e6ffc0e9bfc94623393c.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n amqp_trigger\n SET\n amqp_resource_path = $1,\n queue_name = $2,\n exchange = $3,\n options = $4,\n is_flow = $5,\n edited_by = $6,\n permissioned_as = $7,\n script_path = $8,\n path = $9,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND\n path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "5f2e66e53166840c68efbc521fe9393023e79f948976e6ffc0e9bfc94623393c" +} diff --git a/backend/.sqlx/query-60201b41a0bfea89a201a25ae2431fe94999ca4f11934fbab4638b2653a678dc.json b/backend/.sqlx/query-60201b41a0bfea89a201a25ae2431fe94999ca4f11934fbab4638b2653a678dc.json new file mode 100644 index 0000000000..9f4c0f9bf6 --- /dev/null +++ b/backend/.sqlx/query-60201b41a0bfea89a201a25ae2431fe94999ca4f11934fbab4638b2653a678dc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'admin',\n flow_status = $2 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "60201b41a0bfea89a201a25ae2431fe94999ca4f11934fbab4638b2653a678dc" +} diff --git a/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json b/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json new file mode 100644 index 0000000000..243c3f5fa7 --- /dev/null +++ b/backend/.sqlx/query-620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "620c9efca071cb5fb8b33857129e63954f39b291f525e36e6ece916b98b4d2c9" +} diff --git a/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json index 68d9a1fc3b..8870f18d44 100644 --- a/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json +++ b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json @@ -38,7 +38,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json b/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json new file mode 100644 index 0000000000..9a3cbd6703 --- /dev/null +++ b/backend/.sqlx/query-67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT username FROM usr\n WHERE workspace_id = $1 AND email = $2 AND NOT operator AND NOT disabled", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "67d86d334029276d78bf8fa14b25702eba0b0d674b99f952af94487b3e54211c" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index fcc16e9a7c..0eb26d5dc0 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -162,7 +162,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-694d787e06b5ef9fbb5ca1315652023ecc0b3122501b72ec7f825966419759ac.json b/backend/.sqlx/query-694d787e06b5ef9fbb5ca1315652023ecc0b3122501b72ec7f825966419759ac.json new file mode 100644 index 0000000000..5b69718bfd --- /dev/null +++ b/backend/.sqlx/query-694d787e06b5ef9fbb5ca1315652023ecc0b3122501b72ec7f825966419759ac.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM (\n SELECT ck.job_id FROM concurrency_key ck\n WHERE ck.key = $1 AND ck.ended_at IS NULL\n LIMIT $2\n ) s WHERE EXISTS (\n SELECT 1 FROM v2_job_queue q WHERE q.id = s.job_id AND q.running = false\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "694d787e06b5ef9fbb5ca1315652023ecc0b3122501b72ec7f825966419759ac" +} diff --git a/backend/.sqlx/query-69d0c59bb11a01ffe9934f62feb93c4e64166a6d4d999911dfab743c43bdac35.json b/backend/.sqlx/query-69d0c59bb11a01ffe9934f62feb93c4e64166a6d4d999911dfab743c43bdac35.json new file mode 100644 index 0000000000..a8651b5a73 --- /dev/null +++ b/backend/.sqlx/query-69d0c59bb11a01ffe9934f62feb93c4e64166a6d4d999911dfab743c43bdac35.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT w.parent_workspace_id, w.is_dev_workspace, w.dev_workspace_label,\n p.is_dev_workspace as \"parent_is_dev_workspace?\", p.dev_workspace_label as \"parent_dev_label?\"\n FROM workspace w LEFT JOIN workspace p ON p.id = w.parent_workspace_id\n WHERE w.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "dev_workspace_label", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "parent_is_dev_workspace?", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "parent_dev_label?", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true, + false, + true + ] + }, + "hash": "69d0c59bb11a01ffe9934f62feb93c4e64166a6d4d999911dfab743c43bdac35" +} diff --git a/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json index 43d3b6d7f6..9dbeab4c76 100644 --- a/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json +++ b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json @@ -47,7 +47,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json b/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json index 046d8ad6bd..e4eea7fc8f 100644 --- a/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json +++ b/backend/.sqlx/query-7065f23d04e26831664048f2cfc4f412c57af931f80621aee5012e9cb3535626.json @@ -32,7 +32,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-71faf95db96589e5554882e1dd036bd19923e0b0581efbae448197d6accf01f3.json b/backend/.sqlx/query-71faf95db96589e5554882e1dd036bd19923e0b0581efbae448197d6accf01f3.json new file mode 100644 index 0000000000..76c7a0baa7 --- /dev/null +++ b/backend/.sqlx/query-71faf95db96589e5554882e1dd036bd19923e0b0581efbae448197d6accf01f3.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(\n git_sync,\n '{repositories}',\n (SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' IN ($2, '$res:' || $2)\n THEN CASE WHEN $3::text IS NULL THEN elem - 'open_pr_error'\n ELSE jsonb_set(elem, '{open_pr_error}', to_jsonb($3::text), true) END\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem)\n )\n WHERE workspace_id = $1\n AND jsonb_typeof(git_sync->'repositories') = 'array'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "71faf95db96589e5554882e1dd036bd19923e0b0581efbae448197d6accf01f3" +} diff --git a/backend/.sqlx/query-72b44f728dea65b0542cb72244259703e5029536898469a484ebed23d6226e82.json b/backend/.sqlx/query-72b44f728dea65b0542cb72244259703e5029536898469a484ebed23d6226e82.json new file mode 100644 index 0000000000..85f309bedc --- /dev/null +++ b/backend/.sqlx/query-72b44f728dea65b0542cb72244259703e5029536898469a484ebed23d6226e82.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2 AND enabled IS TRUE", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "72b44f728dea65b0542cb72244259703e5029536898469a484ebed23d6226e82" +} diff --git a/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json b/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json index 7b7e68ad45..23dd32b094 100644 --- a/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json +++ b/backend/.sqlx/query-73fdd01bad58b8be1a52f89faef8d92a983470adcd3cc850734960c905e61e83.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-750eb7365e2590a42d9f3c4cbbf193ea4129874c36e2c334e9e73f4b4c54ad07.json b/backend/.sqlx/query-750eb7365e2590a42d9f3c4cbbf193ea4129874c36e2c334e9e73f4b4c54ad07.json new file mode 100644 index 0000000000..17f12041c4 --- /dev/null +++ b/backend/.sqlx/query-750eb7365e2590a42d9f3c4cbbf193ea4129874c36e2c334e9e73f4b4c54ad07.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, $2, 'other')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "750eb7365e2590a42d9f3c4cbbf193ea4129874c36e2c334e9e73f4b4c54ad07" +} diff --git a/backend/.sqlx/query-7d2923f940cf5a8cbbdc51de91c7e00942a8f4f5251502ef8efc0f510a857551.json b/backend/.sqlx/query-7d2923f940cf5a8cbbdc51de91c7e00942a8f4f5251502ef8efc0f510a857551.json new file mode 100644 index 0000000000..7967a56868 --- /dev/null +++ b/backend/.sqlx/query-7d2923f940cf5a8cbbdc51de91c7e00942a8f4f5251502ef8efc0f510a857551.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed\n SET status = 'canceled', canceled_by = 'monitor', canceled_reason = 'zombie flow',\n flow_status = $2\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "7d2923f940cf5a8cbbdc51de91c7e00942a8f4f5251502ef8efc0f510a857551" +} diff --git a/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json b/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json new file mode 100644 index 0000000000..7eef899aa7 --- /dev/null +++ b/backend/.sqlx/query-7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings\n (workspace_id, color, error_handler_fallback_to_instance_alerts)\n VALUES ($1, $2, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "7da2e04f64cef6634256d7b3b12d8a46dc7320e7dcdd64befe808dec8cbe2265" +} diff --git a/backend/.sqlx/query-7ef0e3f0fdbda1ec514a621c73782f9afac8b6db79668cbdb188719c5a5de6da.json b/backend/.sqlx/query-7ef0e3f0fdbda1ec514a621c73782f9afac8b6db79668cbdb188719c5a5de6da.json new file mode 100644 index 0000000000..fd30017ca3 --- /dev/null +++ b/backend/.sqlx/query-7ef0e3f0fdbda1ec514a621c73782f9afac8b6db79668cbdb188719c5a5de6da.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO concurrency_key (key, job_id) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7ef0e3f0fdbda1ec514a621c73782f9afac8b6db79668cbdb188719c5a5de6da" +} diff --git a/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json b/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json index cd5e361f15..19264cfdfe 100644 --- a/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json +++ b/backend/.sqlx/query-7fbf72d9059fcd77e4c1112fa4fa22e4276c1da653475628889ce17dc904fbaa.json @@ -30,7 +30,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-824464f9d9b39980fee7d9fe8c6f697438aaa8de6233756a3227cb1e4b2fef4e.json b/backend/.sqlx/query-824464f9d9b39980fee7d9fe8c6f697438aaa8de6233756a3227cb1e4b2fef4e.json new file mode 100644 index 0000000000..5ab90a6a4e --- /dev/null +++ b/backend/.sqlx/query-824464f9d9b39980fee7d9fe8c6f697438aaa8de6233756a3227cb1e4b2fef4e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET is_dev_workspace = false,\n parent_workspace_id = CASE WHEN id LIKE 'wm-fork-%' THEN parent_workspace_id ELSE NULL END\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "824464f9d9b39980fee7d9fe8c6f697438aaa8de6233756a3227cb1e4b2fef4e" +} diff --git a/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json b/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json index 1c0e5a0f1b..c2d9b482b5 100644 --- a/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json +++ b/backend/.sqlx/query-836bac47d89113d90bd03a471446eb9016207975af1e37042d81df8cb6ae2c53.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-856697038686ec82f1925f70c57d41e538012b79637c9c0dcd29345ded09f699.json b/backend/.sqlx/query-856697038686ec82f1925f70c57d41e538012b79637c9c0dcd29345ded09f699.json new file mode 100644 index 0000000000..067c1f31b4 --- /dev/null +++ b/backend/.sqlx/query-856697038686ec82f1925f70c57d41e538012b79637c9c0dcd29345ded09f699.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH ref AS (\n SELECT worker_instance, worker_group FROM worker_ping WHERE worker = $1 LIMIT 1\n )\n SELECT\n wp.worker AS \"worker!\",\n wp.ping_at AS \"ping_at!\",\n wp.memory_usage,\n wp.wm_memory_usage,\n wp.memory AS memory_total,\n wp.worker_group,\n wp.worker_instance,\n EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))::float8 AS ping_delta_secs\n FROM worker_ping wp, ref\n WHERE wp.worker <> $1\n AND (\n (ref.worker_instance IS NOT NULL AND wp.worker_instance = ref.worker_instance)\n OR (ref.worker_group IS NOT NULL AND wp.worker_group = ref.worker_group)\n )\n AND wp.ping_at >= $2::timestamptz - interval '5 seconds'\n AND wp.ping_at <= $2::timestamptz + interval '15 seconds'\n ORDER BY ABS(EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))) ASC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "worker!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "ping_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "memory_usage", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "wm_memory_usage", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "memory_total", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "worker_instance", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "ping_delta_secs", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "Text", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + false, + false, + null + ] + }, + "hash": "856697038686ec82f1925f70c57d41e538012b79637c9c0dcd29345ded09f699" +} diff --git a/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json b/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json index 9b3fd8f205..71d71e7843 100644 --- a/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json +++ b/backend/.sqlx/query-87564a196a1662f524407d853db506bf08c28efe82b68b3d44bafbd3d0e91c29.json @@ -38,7 +38,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json b/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json new file mode 100644 index 0000000000..21679c4ec2 --- /dev/null +++ b/backend/.sqlx/query-88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "error_handler_fallback_to_instance_alerts", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "88943f52672bffc69c75bc6b06a6a8e333912b6a5eba1c5d7b3a0ee75cc95b10" +} diff --git a/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json b/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json new file mode 100644 index 0000000000..7cfe9aa070 --- /dev/null +++ b/backend/.sqlx/query-88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace, workspace.dev_workspace_label,\n workspace.owner AS \"created_by?\",\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "username", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "color", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "dev_workspace_label", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "created_by?", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "operator_settings", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + false, + true, + false, + null, + false + ] + }, + "hash": "88a134e4ca82d5ce0334977c7713021ae3e99a5a61ea1c944c1df1368746dfa5" +} diff --git a/backend/.sqlx/query-8e20b696db308f780744d775bc257a9b31539b7fd146e0cff4aa701d06cc6846.json b/backend/.sqlx/query-8e20b696db308f780744d775bc257a9b31539b7fd146e0cff4aa701d06cc6846.json new file mode 100644 index 0000000000..978c4646ae --- /dev/null +++ b/backend/.sqlx/query-8e20b696db308f780744d775bc257a9b31539b7fd146e0cff4aa701d06cc6846.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->>'branch' FROM resource WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e20b696db308f780744d775bc257a9b31539b7fd146e0cff4aa701d06cc6846" +} diff --git a/backend/.sqlx/query-8f163ee5adf4caaaa12a5698e68c749524f1db10a51f3de6cadd4826e6c1d422.json b/backend/.sqlx/query-8f163ee5adf4caaaa12a5698e68c749524f1db10a51f3de6cadd4826e6c1d422.json index 90031c14d6..4e63ce0324 100644 --- a/backend/.sqlx/query-8f163ee5adf4caaaa12a5698e68c749524f1db10a51f3de6cadd4826e6c1d422.json +++ b/backend/.sqlx/query-8f163ee5adf4caaaa12a5698e68c749524f1db10a51f3de6cadd4826e6c1d422.json @@ -36,7 +36,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-90da952a912de746abee641cc4268644853987e62cbcedc8d3a46626a60c9fb0.json b/backend/.sqlx/query-90da952a912de746abee641cc4268644853987e62cbcedc8d3a46626a60c9fb0.json new file mode 100644 index 0000000000..a67728cc90 --- /dev/null +++ b/backend/.sqlx/query-90da952a912de746abee641cc4268644853987e62cbcedc8d3a46626a60c9fb0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE amqp_trigger SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "90da952a912de746abee641cc4268644853987e62cbcedc8d3a46626a60c9fb0" +} diff --git a/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json b/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json new file mode 100644 index 0000000000..10de699b74 --- /dev/null +++ b/backend/.sqlx/query-931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id FROM workspace\n WHERE id = $1 AND owner = $2 AND parent_workspace_id IS NOT NULL AND NOT deleted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "931b2a18250879b0bbd0bec2c88c5aed4273f9f3977144d7c564be07504af9af" +} diff --git a/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json b/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json index c76a069f49..ea5a2946f7 100644 --- a/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json +++ b/backend/.sqlx/query-940b6d78bab940a37a42492f030d2393e297043e4e58555d872b5c4dd89c196a.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json b/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json deleted file mode 100644 index 58f8729225..0000000000 --- a/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM ai_chat_usage WHERE created_at < NOW() - INTERVAL '60 days'", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f" -} diff --git a/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json index 87655ad8c1..52590474a3 100644 --- a/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json +++ b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json @@ -35,7 +35,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index ab01730c02..fc1e28642b 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -36,7 +36,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-a47f7d625a96ad47277609ae173c408eda4323ec12ade20ec0da565051128f26.json b/backend/.sqlx/query-a47f7d625a96ad47277609ae173c408eda4323ec12ade20ec0da565051128f26.json new file mode 100644 index 0000000000..9361eb51aa --- /dev/null +++ b/backend/.sqlx/query-a47f7d625a96ad47277609ae173c408eda4323ec12ade20ec0da565051128f26.json @@ -0,0 +1,100 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM amqp_trigger WHERE workspace_id = $1) AS \"amqp_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\",\n EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS \"gcp_used!\",\n EXISTS(SELECT 1 FROM azure_trigger WHERE workspace_id = $1) AS \"azure_used!\",\n EXISTS(SELECT 1 FROM email_trigger WHERE workspace_id = $1) AS \"email_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'nextcloud'::native_trigger_service) AS \"nextcloud_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'google'::native_trigger_service) AS \"google_used!\",\n EXISTS(SELECT 1 FROM native_trigger WHERE workspace_id = $1 AND service_name = 'github'::native_trigger_service) AS \"github_used!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "websocket_used!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "http_routes_used!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "kafka_used!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "nats_used!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "postgres_used!", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "mqtt_used!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "amqp_used!", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "sqs_used!", + "type_info": "Bool" + }, + { + "ordinal": 8, + "name": "gcp_used!", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "azure_used!", + "type_info": "Bool" + }, + { + "ordinal": 10, + "name": "email_used!", + "type_info": "Bool" + }, + { + "ordinal": 11, + "name": "nextcloud_used!", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "google_used!", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "github_used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "a47f7d625a96ad47277609ae173c408eda4323ec12ade20ec0da565051128f26" +} diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 405904604a..4824bba526 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -192,7 +192,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-a5fbc70721ea71796cf8e180688c45a1eed3aec6d60de7bc894b78189c76da72.json b/backend/.sqlx/query-a5fbc70721ea71796cf8e180688c45a1eed3aec6d60de7bc894b78189c76da72.json new file mode 100644 index 0000000000..712cc41e54 --- /dev/null +++ b/backend/.sqlx/query-a5fbc70721ea71796cf8e180688c45a1eed3aec6d60de7bc894b78189c76da72.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args->'__git_sync_pr_check' AS \"pr\", args->'__git_sync_deploy_check' AS \"deploy\",\n args->>'repo_url_resource_path' AS \"repo_path\"\n FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pr", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "deploy", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "repo_path", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "a5fbc70721ea71796cf8e180688c45a1eed3aec6d60de7bc894b78189c76da72" +} diff --git a/backend/.sqlx/query-add9dcf335bf5d6a53fd6c77db1e6dc47b9e448df2463b7980d689f5dedc39a2.json b/backend/.sqlx/query-add9dcf335bf5d6a53fd6c77db1e6dc47b9e448df2463b7980d689f5dedc39a2.json new file mode 100644 index 0000000000..d005f170f2 --- /dev/null +++ b/backend/.sqlx/query-add9dcf335bf5d6a53fd6c77db1e6dc47b9e448df2463b7980d689f5dedc39a2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO data_metric (workspace_id, script_path, table_path, kind, name, expr, filter)\n SELECT $2, script_path, table_path, kind, name, expr, filter\n FROM data_metric WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "add9dcf335bf5d6a53fd6c77db1e6dc47b9e448df2463b7980d689f5dedc39a2" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index 092d15e592..3bbd0f043e 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -167,7 +167,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json b/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json index 7325a2a313..6efaff7069 100644 --- a/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json +++ b/backend/.sqlx/query-b3f0595cacba194e08b9a3e244d9e637e9e156cd85b69126c87dfff89a47711d.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json b/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json deleted file mode 100644 index ed4288ea5c..0000000000 --- a/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4)\n ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc" -} diff --git a/backend/.sqlx/query-b6cfb752675a3f36975e6cc6c454267d67f58f9cbff0d164b4df45de885b0e9d.json b/backend/.sqlx/query-b6cfb752675a3f36975e6cc6c454267d67f58f9cbff0d164b4df45de885b0e9d.json index da7e3d5787..98bcf5a2c2 100644 --- a/backend/.sqlx/query-b6cfb752675a3f36975e6cc6c454267d67f58f9cbff0d164b4df45de885b0e9d.json +++ b/backend/.sqlx/query-b6cfb752675a3f36975e6cc6c454267d67f58f9cbff0d164b4df45de885b0e9d.json @@ -47,7 +47,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-ba6ffe7fb86878e23f69e9c7636b4da42c00d072a908412a214ab231df784b8f.json b/backend/.sqlx/query-ba6ffe7fb86878e23f69e9c7636b4da42c00d072a908412a214ab231df784b8f.json new file mode 100644 index 0000000000..267efe9e63 --- /dev/null +++ b/backend/.sqlx/query-ba6ffe7fb86878e23f69e9c7636b4da42c00d072a908412a214ab231df784b8f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT COALESCE(dev_workspace_label, 'dev') as \"label!\"\n FROM workspace\n WHERE parent_workspace_id = $1 AND is_dev_workspace AND NOT deleted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "label!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ba6ffe7fb86878e23f69e9c7636b4da42c00d072a908412a214ab231df784b8f" +} diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index 419ab26383..fb35992691 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -81,7 +81,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json b/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json new file mode 100644 index 0000000000..a0697321be --- /dev/null +++ b/backend/.sqlx/query-be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, parent_workspace_id)\n VALUES ($1, $1, 'test-user', 'test-workspace')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "be21088e8b88e01b50a544f220e27d1e30b63c946a4eb640e55850100c3edd3c" +} diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index b9d33a6b5f..809deb1a55 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -112,7 +112,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json b/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json new file mode 100644 index 0000000000..cde9284501 --- /dev/null +++ b/backend/.sqlx/query-be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, is_admin FROM usr where username = $1 AND workspace_id = $2 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "be927c5174436cb9574bacc7cb4e3f746cc8f1909f94a15bd8cde33ce974caa7" +} diff --git a/backend/.sqlx/query-bfbc368c751c8532792ae0a77ed570a14d22ed2a776efa111ad53aa9e77eadf1.json b/backend/.sqlx/query-bfbc368c751c8532792ae0a77ed570a14d22ed2a776efa111ad53aa9e77eadf1.json new file mode 100644 index 0000000000..1d92609205 --- /dev/null +++ b/backend/.sqlx/query-bfbc368c751c8532792ae0a77ed570a14d22ed2a776efa111ad53aa9e77eadf1.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.workspace_id, ws.git_sync\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE NOT w.deleted\n AND ws.git_sync IS NOT NULL\n AND ws.git_sync->'repositories' @> '[{\"auto_pull\": {\"enabled\": true}}]'::jsonb", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bfbc368c751c8532792ae0a77ed570a14d22ed2a776efa111ad53aa9e77eadf1" +} diff --git a/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json b/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json new file mode 100644 index 0000000000..fa96843cc9 --- /dev/null +++ b/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH per_entity AS (\n SELECT feature, kind, key, entity_id,\n SUM(value)::BIGINT AS value,\n MAX(day) AS last_day\n FROM feature_usage\n WHERE day > CURRENT_DATE - 30\n GROUP BY feature, kind, key, entity_id\n )\n SELECT feature, kind, key,\n COUNT(*)::BIGINT AS \"entity_count!\",\n COALESCE(SUM(value), 0)::BIGINT AS \"total_value!\",\n COALESCE(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"median_value!\",\n COALESCE(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"p90_value!\",\n (COUNT(*) FILTER (WHERE last_day < CURRENT_DATE - 3))::BIGINT AS \"inactive_3d_entity_count!\"\n FROM per_entity\n GROUP BY feature, kind, key\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "feature", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "entity_count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "total_value!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "median_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "p90_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "inactive_3d_entity_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + null, + null, + null, + null, + null + ] + }, + "hash": "c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4" +} diff --git a/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json index 8a6ab29126..2bc2bdf950 100644 --- a/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json +++ b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json @@ -34,7 +34,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json index 161b9d36f9..52830d5c73 100644 --- a/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json +++ b/backend/.sqlx/query-c8fb2a1491f90951a6d2e4e9c56d288eb60f6c6644a73cdf949de0159215a7f7.json @@ -43,7 +43,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-c92c08a4b4c06d087c2590081db78f7253e4470287810d7ee13b8519f54b997f.json b/backend/.sqlx/query-c92c08a4b4c06d087c2590081db78f7253e4470287810d7ee13b8519f54b997f.json new file mode 100644 index 0000000000..46ce3af568 --- /dev/null +++ b/backend/.sqlx/query-c92c08a4b4c06d087c2590081db78f7253e4470287810d7ee13b8519f54b997f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.git_sync FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id\n WHERE ws.workspace_id = $1 AND NOT w.deleted", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "git_sync", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c92c08a4b4c06d087c2590081db78f7253e4470287810d7ee13b8519f54b997f" +} diff --git a/backend/.sqlx/query-a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2.json b/backend/.sqlx/query-cb26dc8e09c0525963c3d9d316e6cf0437c4efcdeae45fba0f240fc1175bf003.json similarity index 68% rename from backend/.sqlx/query-a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2.json rename to backend/.sqlx/query-cb26dc8e09c0525963c3d9d316e6cf0437c4efcdeae45fba0f240fc1175bf003.json index daeaccb715..dce702a619 100644 --- a/backend/.sqlx/query-a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2.json +++ b/backend/.sqlx/query-cb26dc8e09c0525963c3d9d316e6cf0437c4efcdeae45fba0f240fc1175bf003.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2 AND (is_public IS true OR created_by = $4)", "describe": { "columns": [ { @@ -13,12 +13,13 @@ "Left": [ "Uuid", "Text", - "Bool" + "Bool", + "Text" ] }, "nullable": [ null ] }, - "hash": "a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2" + "hash": "cb26dc8e09c0525963c3d9d316e6cf0437c4efcdeae45fba0f240fc1175bf003" } diff --git a/backend/.sqlx/query-cb792a399cc1a7eb67222380cc0b2fb39d4ab8278cd605219541d06837e974d2.json b/backend/.sqlx/query-cb792a399cc1a7eb67222380cc0b2fb39d4ab8278cd605219541d06837e974d2.json new file mode 100644 index 0000000000..756e94cb41 --- /dev/null +++ b/backend/.sqlx/query-cb792a399cc1a7eb67222380cc0b2fb39d4ab8278cd605219541d06837e974d2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM data_metric WHERE workspace_id = $1 AND (script_path = $2 OR script_path = $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cb792a399cc1a7eb67222380cc0b2fb39d4ab8278cd605219541d06837e974d2" +} diff --git a/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json index f567c462f8..0cf9c77f62 100644 --- a/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json +++ b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-d1b882ab87de6d6cd16e8bc4364e6f11e55231d2e62b0f8a7404f0e6093d7d68.json b/backend/.sqlx/query-d1b882ab87de6d6cd16e8bc4364e6f11e55231d2e62b0f8a7404f0e6093d7d68.json new file mode 100644 index 0000000000..97d811b347 --- /dev/null +++ b/backend/.sqlx/query-d1b882ab87de6d6cd16e8bc4364e6f11e55231d2e62b0f8a7404f0e6093d7d68.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 AND suspend > 0", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d1b882ab87de6d6cd16e8bc4364e6f11e55231d2e62b0f8a7404f0e6093d7d68" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index d97c02d26b..2660ca05ef 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -112,7 +112,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index 052d83fcd9..0bd066afd1 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -252,7 +252,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json index eb340e6cc7..049856cbde 100644 --- a/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json +++ b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json @@ -28,7 +28,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json b/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json index e60f5cc187..317f33de07 100644 --- a/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json +++ b/backend/.sqlx/query-d4e0da9f9653d532770066310f85e59e5edda1faea72f39603814afb6a3cd596.json @@ -48,7 +48,8 @@ "trigger_nextcloud", "trigger_google", "trigger_github", - "data_pipeline" + "data_pipeline", + "trigger_amqp" ] } } diff --git a/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json b/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json new file mode 100644 index 0000000000..13e31508a0 --- /dev/null +++ b/backend/.sqlx/query-d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d6269058b3e5146de5b32ac2363ad84b0a1bb18b35e0f505ab008bff1310528b" +} diff --git a/backend/.sqlx/query-d977778801cc5efdfc4051d5b77d75eda5fc20669d598f0dbbb301030e58d702.json b/backend/.sqlx/query-d977778801cc5efdfc4051d5b77d75eda5fc20669d598f0dbbb301030e58d702.json new file mode 100644 index 0000000000..420092f7e8 --- /dev/null +++ b/backend/.sqlx/query-d977778801cc5efdfc4051d5b77d75eda5fc20669d598f0dbbb301030e58d702.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM (\n SELECT 1 FROM v2_job_queue\n WHERE workspace_id = $1 AND running = false\n LIMIT $2\n ) s", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d977778801cc5efdfc4051d5b77d75eda5fc20669d598f0dbbb301030e58d702" +} diff --git a/backend/.sqlx/query-d9d997591a163f25a9be5134bffacd0068c51ca21018d6751cc1044e38c06730.json b/backend/.sqlx/query-d9d997591a163f25a9be5134bffacd0068c51ca21018d6751cc1044e38c06730.json new file mode 100644 index 0000000000..d5e5970903 --- /dev/null +++ b/backend/.sqlx/query-d9d997591a163f25a9be5134bffacd0068c51ca21018d6751cc1044e38c06730.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO amqp_trigger (\n amqp_resource_path, queue_name, exchange, options, path, script_path, is_flow,\n workspace_id, edited_by, edited_at, extra_perms, server_id, last_server_ping,\n error, error_handler_path, error_handler_args, retry, mode, permissioned_as, labels\n )\n SELECT\n amqp_resource_path, queue_name, exchange, options, path, script_path, is_flow,\n $1, edited_by, edited_at, extra_perms, NULL, NULL,\n NULL, error_handler_path, error_handler_args, retry, 'disabled'::TRIGGER_MODE, permissioned_as, labels\n FROM amqp_trigger WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d9d997591a163f25a9be5134bffacd0068c51ca21018d6751cc1044e38c06730" +} diff --git a/backend/.sqlx/query-dcfbbe009dccc40249bbab1e6ae378985f6e7a593361d1a185ed606db13d84d4.json b/backend/.sqlx/query-dcfbbe009dccc40249bbab1e6ae378985f6e7a593361d1a185ed606db13d84d4.json new file mode 100644 index 0000000000..23ab7ae78a --- /dev/null +++ b/backend/.sqlx/query-dcfbbe009dccc40249bbab1e6ae378985f6e7a593361d1a185ed606db13d84d4.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO data_metric (workspace_id, script_path, table_path, kind, name, expr, filter) SELECT $1, $2, $3, k, n, e, f FROM UNNEST($4::text[], $5::text[], $6::text[], $7::text[]) AS t(k, n, e, f)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "dcfbbe009dccc40249bbab1e6ae378985f6e7a593361d1a185ed606db13d84d4" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index a35373a959..ad34773e36 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -192,7 +192,8 @@ "github", "azure", "asset", - "freshness" + "freshness", + "amqp" ] } } diff --git a/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json index 3f403a8f95..cb0b92a822 100644 --- a/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json +++ b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json @@ -34,7 +34,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-ea1895df5a6afa5af59c3771291c85a1166294ea655e934baf5e3e1daa0fac1a.json b/backend/.sqlx/query-ea1895df5a6afa5af59c3771291c85a1166294ea655e934baf5e3e1daa0fac1a.json new file mode 100644 index 0000000000..206614b73e --- /dev/null +++ b/backend/.sqlx/query-ea1895df5a6afa5af59c3771291c85a1166294ea655e934baf5e3e1daa0fac1a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE workspace_settings\n SET git_sync = jsonb_set(\n git_sync,\n '{repositories}',\n (SELECT jsonb_agg(\n CASE WHEN elem->>'git_repo_resource_path' = $2\n THEN jsonb_set(\n jsonb_set(\n jsonb_set(elem, '{auto_pull}',\n COALESCE(elem->'auto_pull', '{\"enabled\": false}'::jsonb), true),\n '{auto_pull,last_synced_sha}', $3, true),\n '{auto_pull,last_pull_status}', $4, true)\n ELSE elem END)\n FROM jsonb_array_elements(git_sync->'repositories') AS elem)\n )\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ea1895df5a6afa5af59c3771291c85a1166294ea655e934baf5e3e1daa0fac1a" +} diff --git a/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json index b48f144f8e..461b7fc037 100644 --- a/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json +++ b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-ed42846826aa3f82f17097e52fcbf55f4e0a37d7c9280064a5636d9d3bf1f6c2.json b/backend/.sqlx/query-ed42846826aa3f82f17097e52fcbf55f4e0a37d7c9280064a5636d9d3bf1f6c2.json new file mode 100644 index 0000000000..9b5c969f66 --- /dev/null +++ b/backend/.sqlx/query-ed42846826aa3f82f17097e52fcbf55f4e0a37d7c9280064a5636d9d3bf1f6c2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (kind::text NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND parent_job IS NULL) AS \"is_wac!\"\n FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_wac!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ed42846826aa3f82f17097e52fcbf55f4e0a37d7c9280064a5636d9d3bf1f6c2" +} diff --git a/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json index c28997d702..902a07854d 100644 --- a/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json +++ b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json @@ -27,7 +27,8 @@ "nextcloud", "google", "github", - "azure" + "azure", + "amqp" ] } } diff --git a/backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json b/backend/.sqlx/query-ef57c1622cc57e4485ffe086dd55f62df8b7f510b4c478818ff8ab1a11d7d5ea.json similarity index 75% rename from backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json rename to backend/.sqlx/query-ef57c1622cc57e4485ffe086dd55f62df8b7f510b4c478818ff8ab1a11d7d5ea.json index f9d007d4c1..bd2d3cefda 100644 --- a/backend/.sqlx/query-55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5.json +++ b/backend/.sqlx/query-ef57c1622cc57e4485ffe086dd55f62df8b7f510b4c478818ff8ab1a11d7d5ea.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2 AND (is_public IS true OR created_by = $4)", "describe": { "columns": [ { @@ -13,12 +13,13 @@ "Left": [ "Uuid", "Text", - "Bool" + "Bool", + "Text" ] }, "nullable": [ null ] }, - "hash": "55002cccf17e32af5b077a17707122cfeeeebf5a9392c6798a486274d8d233d5" + "hash": "ef57c1622cc57e4485ffe086dd55f62df8b7f510b4c478818ff8ab1a11d7d5ea" } diff --git a/backend/.sqlx/query-f02977ee5df3d8f734fb32836583abaf747aef7078ae57b0373f06c70b6f2f9f.json b/backend/.sqlx/query-f02977ee5df3d8f734fb32836583abaf747aef7078ae57b0373f06c70b6f2f9f.json new file mode 100644 index 0000000000..f449de1836 --- /dev/null +++ b/backend/.sqlx/query-f02977ee5df3d8f734fb32836583abaf747aef7078ae57b0373f06c70b6f2f9f.json @@ -0,0 +1,82 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n args->'__git_sync_open_pr' as \"marker\",\n args->>'repo_url_resource_path' as \"repo_path\",\n args->>'parent_workspace_id' as \"parent_workspace_id\",\n args->>'dev_workspace_label' as \"dev_workspace_label\",\n args->>'parent_dev_workspace_label' as \"parent_dev_workspace_label\",\n COALESCE((args->'use_individual_branch')::bool, false) as \"use_individual_branch!\",\n COALESCE((args->'group_by_folder')::bool, false) as \"group_by_folder!\",\n COALESCE(args->'items'->0->>'path', args->>'path', '') as \"item_path!\",\n COALESCE(args->'items'->0->>'parent_path', args->>'parent_path', '') as \"item_parent_path!\",\n COALESCE(args->'items'->0->>'path_type', args->>'path_type', '') as \"path_type!\",\n COALESCE(args->'items'->0->>'commit_msg', args->>'commit_msg', '') as \"commit_msg!\"\n FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "marker", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "repo_path", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "parent_workspace_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "dev_workspace_label", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "parent_dev_workspace_label", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "use_individual_branch!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "group_by_folder!", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "item_path!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "item_parent_path!", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "path_type!", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "commit_msg!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "f02977ee5df3d8f734fb32836583abaf747aef7078ae57b0373f06c70b6f2f9f" +} diff --git a/backend/.sqlx/query-f3bae157b92cbc1b3741593a620c9203012a7d3754f56af665e2f34a969eabd7.json b/backend/.sqlx/query-f3bae157b92cbc1b3741593a620c9203012a7d3754f56af665e2f34a969eabd7.json new file mode 100644 index 0000000000..f787574b3a --- /dev/null +++ b/backend/.sqlx/query-f3bae157b92cbc1b3741593a620c9203012a7d3754f56af665e2f34a969eabd7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f3bae157b92cbc1b3741593a620c9203012a7d3754f56af665e2f34a969eabd7" +} diff --git a/backend/.sqlx/query-fa1b9fcdd344fa2c88a187f4a558e375d93caf76c4cec362c2b92d2a7b8704a2.json b/backend/.sqlx/query-fa1b9fcdd344fa2c88a187f4a558e375d93caf76c4cec362c2b92d2a7b8704a2.json new file mode 100644 index 0000000000..0d9e4ac3ad --- /dev/null +++ b/backend/.sqlx/query-fa1b9fcdd344fa2c88a187f4a558e375d93caf76c4cec362c2b92d2a7b8704a2.json @@ -0,0 +1,83 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",\n j.kind AS \"job_kind!: JobKind\", c.canceled_by,\n COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\",\n j.raw_flow AS \"raw_flow: Json>\"\n FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_hash: ScriptHash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "job_kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlestepflow", + "flowscript", + "flownode", + "appscript", + "aiagent", + "unassigned_script", + "unassigned_flow", + "unassigned_singlestepflow" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "flow_status: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 5, + "name": "raw_flow: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + true, + false, + true, + null, + true + ] + }, + "hash": "fa1b9fcdd344fa2c88a187f4a558e375d93caf76c4cec362c2b92d2a7b8704a2" +} diff --git a/backend/.sqlx/query-ffbe9fe78a7fc0e95a5a29c8d17f4367ef6b6cc3de20da6ef8c98f679b832240.json b/backend/.sqlx/query-ffbe9fe78a7fc0e95a5a29c8d17f4367ef6b6cc3de20da6ef8c98f679b832240.json new file mode 100644 index 0000000000..9320a9625f --- /dev/null +++ b/backend/.sqlx/query-ffbe9fe78a7fc0e95a5a29c8d17f4367ef6b6cc3de20da6ef8c98f679b832240.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value->>'replication_user_pwd' FROM global_settings WHERE name = 'custom_instance_pg_databases';", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "ffbe9fe78a7fc0e95a5a29c8d17f4367ef6b6cc3de20da6ef8c98f679b832240" +} diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 96ad6a9d76..f686e94493 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -60,7 +60,7 @@ only what you need — build time scales with the set. | `license` | License-key/plan plumbing (`LICENSE_KEY`). Pulled in by `enterprise`. Having the feature compiled does **not** require a license *key* at runtime — CE defaults to a free plan and most EE paths still run keyless. | License-gated behavior. | | `parquet` | S3/object-storage support: the `job_helpers/*` and `apps_u/*` S3 endpoints, parquet/CSV preview, workspace large-file storage. Without it those routes return `"requires parquet"`. | Anything touching S3/object storage or datasets. | | `duckdb` | DuckDB script executor (also needs the FFI dylib — see above). | DuckDB scripts, DuckLake. | -| `python` `rust` `php` `java` `ruby` `csharp` `nu` `deno_core` `mysql` `mssql` `bigquery` `oracledb` `rlang` | Each enables that language/DB runtime for job execution. | Running jobs in that language. | +| `python` `rust` `php` `java` `ruby` `csharp` `nu` `deno_core` `mysql` `mssql` `bigquery` `snowflake` `oracledb` `rlang` | Each enables that language/DB runtime for job execution. | Running jobs in that language. | | `mcp` | MCP gateway routes (baseline `quickjs` does NOT include it → MCP routes 404). | MCP work. | | `websocket` `http_trigger` `kafka` `nats` `mqtt_trigger` `sqs_trigger` `gcp_trigger` `azure_trigger` `postgres_trigger` `native_trigger` | Each native trigger kind; none on by default (creating one 404s without its feature). | Working on / exercising that trigger. | | `no_auth` | Treats every request as an admin superadmin (`CLOUD_HOSTED`-guarded). | Local auth-free experiments only. | diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 489390e83a..05435680b0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -128,6 +128,54 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "amq-protocol" +version = "7.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587d313f3a8b4a40f866cc84b6059fe83133bf172165ac3b583129dd211d8e1c" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom", + "serde", +] + +[[package]] +name = "amq-protocol-tcp" +version = "7.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc707ab9aa964a85d9fc25908a3fdc486d2e619406883b3105b48bf304a8d606" +dependencies = [ + "amq-protocol-uri", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "7.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf99351d92a161c61ec6ecb213bc7057f5b837dd4e64ba6cb6491358efd770c4" +dependencies = [ + "cookie-factory", + "nom", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "7.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89f8273826a676282208e5af38461a07fe939def57396af6ad5997fcf56577d" +dependencies = [ + "amq-protocol-types", + "percent-encoding", + "url", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -189,9 +237,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "ar_archive_writer" @@ -472,7 +520,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ - "asn1-rs-derive", + "asn1-rs-derive 0.5.1", "asn1-rs-impl", "displaydoc", "nom", @@ -482,6 +530,22 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive 0.6.0", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.19", + "time", +] + [[package]] name = "asn1-rs-derive" version = "0.5.1" @@ -494,6 +558,18 @@ dependencies = [ "synstructure", ] +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -555,6 +631,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.19" @@ -574,6 +662,92 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand 2.5.0", + "futures-lite 2.6.1", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13f937e26114b93193065fd44f507aa2e9169ad0cdabbb996920b1fe1ddea7ba" +dependencies = [ + "async-channel 2.5.0", + "async-executor", + "async-io 2.6.0", + "async-lock 3.4.2", + "blocking", + "futures-lite 2.6.1", +] + +[[package]] +name = "async-global-executor-trait" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9af57045d58eeb1f7060e7025a1631cbc6399e0a1d10ad6735b3d0ea7f8346ce" +dependencies = [ + "async-global-executor", + "async-trait", + "executor-trait", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.28", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite 2.6.1", + "parking", + "polling 3.11.0", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -646,6 +820,18 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288f83726785267c6f2ef073a3d83dc3f9b81464e9f99898240cced85fce35a" +[[package]] +name = "async-reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6012d170ad00de56c9ee354aef2e358359deb1ec504254e0e5a3774771de0e" +dependencies = [ + "async-io 1.13.0", + "async-trait", + "futures-core", + "reactor-trait", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -680,14 +866,20 @@ dependencies = [ ] [[package]] -name = "async-trait" -version = "0.1.89" +name = "async-task" +version = "4.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -699,7 +891,7 @@ dependencies = [ "async-compression", "chrono", "crc32fast", - "futures-lite", + "futures-lite 2.6.1", "pin-project", "thiserror 1.0.69", "tokio", @@ -759,7 +951,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "hex", "http 1.4.2", "ring 0.17.14", @@ -784,9 +976,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -795,9 +987,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -823,7 +1015,7 @@ dependencies = [ "aws-types", "bytes", "bytes-utils", - "fastrand", + "fastrand 2.5.0", "http 1.4.2", "http-body 1.1.0", "percent-encoding", @@ -849,7 +1041,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -875,7 +1067,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "hyper 0.14.32", "regex-lite", @@ -898,7 +1090,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "once_cell", "regex-lite", @@ -924,7 +1116,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -949,7 +1141,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -972,7 +1164,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "regex-lite", "tracing", @@ -995,7 +1187,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -1019,7 +1211,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "regex-lite", @@ -1043,7 +1235,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "regex-lite", "tracing", @@ -1152,7 +1344,7 @@ dependencies = [ "http 1.4.2", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.9", "hyper-util", @@ -1217,7 +1409,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 2.5.0", "http 0.2.12", "http 1.4.2", "http-body 0.4.6", @@ -1346,7 +1538,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "itoa", "matchit 0.8.4", @@ -1429,7 +1621,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "fastrand", + "fastrand 2.5.0", "gloo-timers", "tokio", ] @@ -1532,7 +1724,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1552,7 +1744,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1595,9 +1787,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] @@ -1690,6 +1882,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel 2.5.0", + "async-task", + "futures-io", + "futures-lite 2.6.1", + "piper", +] + [[package]] name = "bollard" version = "0.18.1" @@ -1704,7 +1909,7 @@ dependencies = [ "hex", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1715,7 +1920,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tower-service", @@ -1761,9 +1966,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3f6da4992df95bbcd9af42a6c7dcb994498fc9048230405f3b36ff7cd3f145" +checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" dependencies = [ "borsh-derive", "bytes", @@ -1772,9 +1977,9 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae8fb4fb5740e4b2c4884ff95f5f32f5e8479db1e8fd8eb49ddbe09eb09bb7c" +checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" dependencies = [ "once_cell", "proc-macro-crate", @@ -1909,9 +2114,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.25.1" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] @@ -2005,7 +2210,7 @@ dependencies = [ "rand_distr", "rayon", "safetensors", - "thiserror 2.0.18", + "thiserror 2.0.19", "yoke", "zip", ] @@ -2023,7 +2228,7 @@ dependencies = [ "rayon", "safetensors", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -2085,9 +2290,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -2124,9 +2329,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -2205,9 +2410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -2215,9 +2420,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -2227,14 +2432,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -2252,6 +2457,18 @@ dependencies = [ "cc", ] +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid", + "der", + "spki", + "x509-cert", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -2419,6 +2636,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -3492,7 +3715,7 @@ dependencies = [ "swc_sourcemap", "swc_visit", "text_lines", - "thiserror 2.0.18", + "thiserror 2.0.19", "unicode-width 0.2.2", "url", ] @@ -3537,7 +3760,7 @@ dependencies = [ "smallvec", "sourcemap", "static_assertions", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", "v8", @@ -3584,7 +3807,7 @@ dependencies = [ "sha2 0.10.9", "signature", "spki", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "uuid", "x25519-dalek", @@ -3668,7 +3891,7 @@ dependencies = [ "hickory-resolver", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.27.9", "hyper-util", "ipnet", @@ -3676,7 +3899,7 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-rustls 0.26.4", "tokio-socks", @@ -3708,7 +3931,7 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "winapi", "windows-sys 0.59.0", ] @@ -3782,7 +4005,7 @@ dependencies = [ "serde", "sha2 0.10.9", "socket2 0.5.10", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-vsock", "url", @@ -3803,7 +4026,7 @@ dependencies = [ "strum", "strum_macros", "syn 2.0.119", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -3815,7 +4038,7 @@ dependencies = [ "deno_error 0.6.1", "percent-encoding", "sys_traits", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", ] @@ -3842,7 +4065,7 @@ dependencies = [ "serde_json", "sys_traits", "temp_deno_which", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", "winapi", "windows-sys 0.59.0", @@ -3854,7 +4077,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bc6b10059f0ccb14c6e0319c5275f0407fb2f9ffe405cd555700561999ea4bf" dependencies = [ - "fastrand", + "fastrand 2.5.0", "futures-channel", "libc", "windows-sys 0.59.0", @@ -3871,7 +4094,7 @@ dependencies = [ "deno_error 0.6.1", "deno_tls", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.27.9", "hyper-util", "log", @@ -3883,7 +4106,7 @@ dependencies = [ "opentelemetry_sdk 0.27.1", "pin-project", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", ] @@ -3911,7 +4134,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "webpki-roots 0.26.11", ] @@ -3954,7 +4177,7 @@ dependencies = [ "flate2", "futures", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "uuid", ] @@ -3975,6 +4198,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", + "der_derive", + "flagset", "pem-rfc7468", "zeroize", ] @@ -3985,7 +4210,7 @@ version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", "displaydoc", "nom", "num-bigint", @@ -3993,6 +4218,31 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs 0.7.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "deranged" version = "0.5.8" @@ -4117,6 +4367,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "digest" version = "0.9.0" @@ -4255,6 +4514,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "doc-comment" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" + [[package]] name = "dotenv" version = "0.15.0" @@ -4584,6 +4849,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "executor-trait" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c39dff9342e4e0e16ce96be751eb21a94e94a87bb2f6e63ad1961c2ce109bf" +dependencies = [ + "async-trait", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -4620,9 +4894,18 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" -version = "2.4.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" @@ -4668,13 +4951,19 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flatbuffers" version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "rustc_version 0.4.1", ] @@ -4817,9 +5106,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -4832,9 +5121,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -4842,15 +5131,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -4870,9 +5159,24 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] [[package]] name = "futures-lite" @@ -4880,7 +5184,7 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" dependencies = [ - "fastrand", + "fastrand 2.5.0", "futures-core", "futures-io", "parking", @@ -4889,9 +5193,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", @@ -4900,21 +5204,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -5206,15 +5510,15 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "globset" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" dependencies = [ "aho-corasick", "bstr", @@ -5301,7 +5605,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebc6e7327e49a66ffb40508c673b7643191bd6b509530193bda97f09272cdcf" dependencies = [ - "async-channel", + "async-channel 1.9.0", "async-stream", "google-cloud-auth", "google-cloud-gax", @@ -5331,7 +5635,7 @@ checksum = "9758a950dc61a15bc65162f72f5bec7e8efde91f102dc7dce7317f67019a6ba9" dependencies = [ "anyhow", "strum", - "thiserror 2.0.18", + "thiserror 2.0.19", "unic-ucd-category", ] @@ -5513,6 +5817,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hermit-abi" version = "0.5.2" @@ -5543,7 +5853,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "ureq", "windows-sys 0.60.2", @@ -5568,7 +5878,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tinyvec", "tokio", "tracing", @@ -5592,7 +5902,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", ] @@ -5732,7 +6042,7 @@ dependencies = [ "futures", "http 1.4.2", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.26.0", "hyper-tls", "hyper-tungstenite", @@ -5782,9 +6092,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -5812,31 +6122,27 @@ dependencies = [ "futures-util", "headers", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.27.9", - "hyper-tls", "hyper-util", - "native-tls", "pin-project-lite", "tokio", - "tokio-native-tls", "tokio-rustls 0.26.4", "tower-service", ] [[package]] name = "hyper-named-pipe" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" dependencies = [ "hex", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "pin-project-lite", "tokio", "tower-service", - "winapi", ] [[package]] @@ -5863,7 +6169,7 @@ checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "log", "rustls 0.22.4", @@ -5881,7 +6187,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "log", "rustls 0.23.35", @@ -5889,7 +6195,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] @@ -5898,7 +6204,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "pin-project-lite", "tokio", @@ -5913,7 +6219,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "native-tls", "tokio", @@ -5928,7 +6234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a343d17fe7885302ed7252767dc7bb83609a874b6ff581142241ec4b73957ad" dependencies = [ "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "pin-project-lite", "tokio", @@ -5948,7 +6254,7 @@ dependencies = [ "futures-util", "http 1.4.2", "http-body 1.1.0", - "hyper 1.10.1", + "hyper 1.11.0", "ipnet", "libc", "percent-encoding", @@ -5969,7 +6275,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-util", "pin-project-lite", "tokio", @@ -6161,6 +6467,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -6176,13 +6491,24 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "io-uring" version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "libc", ] @@ -6354,7 +6680,7 @@ dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6367,7 +6693,7 @@ dependencies = [ "pest_derive", "regex", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6493,7 +6819,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-http-proxy", "hyper-rustls 0.27.9", "hyper-timeout", @@ -6507,7 +6833,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tower 0.5.3", @@ -6531,7 +6857,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6569,7 +6895,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tracing", @@ -6581,6 +6907,28 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +[[package]] +name = "lapin" +version = "2.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d2aa4725b9607915fa1a73e940710a3be6af508ce700e56897cbe8847fbb07" +dependencies = [ + "amq-protocol", + "async-global-executor-trait", + "async-reactor-trait", + "async-trait", + "executor-trait", + "flume", + "futures-core", + "futures-io", + "parking_lot", + "pinky-swear", + "reactor-trait", + "serde", + "tracing", + "waker-fn", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -6655,9 +7003,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libffi-sys" @@ -6674,7 +7022,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e668df13f2e97f3eed52d9301f6b1c4c1ccfccc30eab9e6628e4a8c1fc3546" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "lazy_static", "libgssapi-sys", @@ -6723,7 +7071,7 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "plain", "redox_syscall 0.9.0", @@ -6778,6 +7126,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -6936,7 +7290,7 @@ dependencies = [ "base64 0.21.7", "block-modes", "crc-any", - "des", + "des 0.7.0", "digest 0.9.0", "md-5 0.9.1", "sha2 0.9.9", @@ -7242,7 +7596,7 @@ version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ - "async-lock", + "async-lock 3.4.2", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", @@ -7316,7 +7670,7 @@ dependencies = [ "quote", "syn 2.0.119", "termcolor", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -7341,7 +7695,7 @@ dependencies = [ "rand 0.10.2", "serde", "socket2 0.6.5", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-native-tls", "tokio-util", @@ -7356,7 +7710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "btoi", "byteorder", "bytes", @@ -7373,7 +7727,7 @@ dependencies = [ "serde_json", "sha1", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.19", "uuid", ] @@ -7415,7 +7769,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "libc", ] @@ -7426,7 +7780,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7438,7 +7792,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7450,7 +7804,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -7587,7 +7941,7 @@ dependencies = [ "num-format", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "typetag", "windows-sys 0.48.0", ] @@ -7753,7 +8107,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi", + "hermit-abi 0.5.2", "libc", ] @@ -7829,7 +8183,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.10.1", + "hyper 1.11.0", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -7842,7 +8196,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -7866,7 +8220,16 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs 0.7.2", ] [[package]] @@ -7899,7 +8262,7 @@ version = "6.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "once_cell", "onig_sys", @@ -7958,7 +8321,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -8035,7 +8398,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", ] @@ -8110,7 +8473,7 @@ dependencies = [ "opentelemetry_sdk 0.30.0", "prost", "reqwest 0.12.28", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tonic 0.13.1", "tracing", @@ -8191,7 +8554,7 @@ dependencies = [ "percent-encoding", "rand 0.9.0", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", ] @@ -8277,6 +8640,28 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "p12-keystore" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cae83056e7cb770211494a0ecf66d9fa7eba7d00977e5bb91f0e925b40b937f" +dependencies = [ + "cbc", + "cms", + "der", + "des 0.8.1", + "hex", + "hmac", + "pkcs12", + "pkcs5", + "rand 0.9.0", + "rc2", + "sha1", + "sha2 0.10.9", + "thiserror 2.0.19", + "x509-parser 0.17.0", +] + [[package]] name = "p256" version = "0.13.2" @@ -8418,6 +8803,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac", +] + [[package]] name = "pem" version = "1.1.1" @@ -8466,9 +8861,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" dependencies = [ "memchr", "ucd-trie", @@ -8476,9 +8871,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" dependencies = [ "pest", "pest_generator", @@ -8486,9 +8881,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" dependencies = [ "pest", "pest_meta", @@ -8499,9 +8894,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" dependencies = [ "pest", ] @@ -8658,6 +9053,29 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pinky-swear" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1ea6e230dd3a64d61bcb8b79e597d3ab6b4c94ec7a234ce687dd718b4f2e657" +dependencies = [ + "doc-comment", + "flume", + "parking_lot", + "tracing", +] + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand 2.5.0", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -8669,6 +9087,36 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs12" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" +dependencies = [ + "cms", + "const-oid", + "der", + "digest 0.10.7", + "spki", + "x509-cert", + "zeroize", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes 0.8.3", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -8691,6 +9139,36 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.5.2", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "polyval" version = "0.6.2" @@ -8705,9 +9183,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "postgres-native-tls" @@ -8882,9 +9360,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -8909,7 +9387,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "flate2", "hex", @@ -8923,7 +9401,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "hex", ] @@ -8939,7 +9417,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -9010,7 +9488,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "getopts", "memchr", "unicase", @@ -9091,7 +9569,7 @@ dependencies = [ "rustc-hash 2.1.3", "rustls 0.23.35", "socket2 0.6.5", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "web-time", @@ -9114,7 +9592,7 @@ dependencies = [ "rustls 0.23.35", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.19", "tinyvec", "tracing", "web-time", @@ -9136,9 +9614,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -9304,7 +9782,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9338,6 +9816,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rc2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -9348,7 +9835,7 @@ dependencies = [ "ring 0.17.14", "rustls-pki-types", "time", - "x509-parser", + "x509-parser 0.16.0", "yasna", ] @@ -9386,6 +9873,17 @@ dependencies = [ "sasl2-sys", ] +[[package]] +name = "reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "438a4293e4d097556730f4711998189416232f009c137389e0f961d2bc0ddc58" +dependencies = [ + "async-trait", + "futures-core", + "futures-io", +] + [[package]] name = "reborrow" version = "0.5.5" @@ -9418,7 +9916,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9427,7 +9925,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -9449,34 +9947,34 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -9486,9 +9984,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -9547,7 +10045,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.27.9", "hyper-tls", "hyper-util", @@ -9577,7 +10075,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] @@ -9595,7 +10093,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-rustls 0.27.9", "hyper-util", "js-sys", @@ -9636,7 +10134,7 @@ dependencies = [ "http 1.4.2", "reqwest 0.13.1", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tower-service", ] @@ -9651,11 +10149,11 @@ dependencies = [ "futures", "getrandom 0.2.17", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "reqwest 0.13.1", "reqwest-middleware", "retry-policies", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "wasmtimer", @@ -9769,7 +10267,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -9808,7 +10306,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" dependencies = [ - "async-lock", + "async-lock 3.4.2", "hashbrown 0.16.1", "relative-path", "rquickjs-sys", @@ -9988,13 +10486,27 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "0.37.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519165d378b97752ca44bbe15047d5d3409e875f39327546b42ac81d7e18c1b6" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -10007,7 +10519,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -10056,6 +10568,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-connector" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70cc376c6ba1823ae229bacf8ad93c136d93524eab0e4e5e0e4f96b9c4e5b212" +dependencies = [ + "log", + "rustls 0.23.35", + "rustls-native-certs 0.7.3", + "rustls-pki-types", + "rustls-webpki 0.103.13", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -10113,9 +10638,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -10279,6 +10804,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "samael" version = "0.0.20" @@ -10300,7 +10834,7 @@ dependencies = [ "quick-xml", "rand 0.9.0", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", "uuid", ] @@ -10415,6 +10949,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sct" version = "0.7.1" @@ -10461,7 +11006,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -10474,7 +11019,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -10520,9 +11065,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -10573,22 +11118,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -10604,9 +11149,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap 2.14.0", "itoa", @@ -10647,13 +11192,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -10687,7 +11232,7 @@ dependencies = [ "num-bigint", "serde", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.19", "v8", ] @@ -10913,7 +11458,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", ] @@ -10978,9 +11523,19 @@ checksum = "8003eb09806ff2ae4661dd0dca27cbd9f65ba85de06cc0302c364b0d661ba368" [[package]] name = "snap" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "socket2" @@ -11154,7 +11709,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tracing", @@ -11210,7 +11765,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "bytes", "chrono", @@ -11240,7 +11795,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "uuid", "whoami", @@ -11255,7 +11810,7 @@ dependencies = [ "atoi", "base64 0.22.1", "bigdecimal", - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "chrono", "crc", @@ -11281,7 +11836,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "uuid", "whoami", @@ -11307,7 +11862,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "url", "uuid", @@ -11315,9 +11870,9 @@ dependencies = [ [[package]] name = "sse-stream" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39f24a9b78c40b90817bbcd1821c74ddfd74916aadd29403d001532a9195532d" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" dependencies = [ "bytes", "futures-util", @@ -11530,7 +12085,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "is-macro", "num-bigint", "once_cell", @@ -11586,7 +12141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ "arrayvec", - "bitflags 2.13.0", + "bitflags 2.13.1", "either", "num-bigint", "phf 0.11.3", @@ -11853,6 +12408,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -11908,7 +12474,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01198a2debb237c62b6826ec7081082d951f46dbb64b0e8c7649a452230d1dfc" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "enum-as-inner", "libc", @@ -11936,7 +12502,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -12018,7 +12584,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", "typetag", "uuid", @@ -12130,6 +12696,18 @@ dependencies = [ "xattr", ] +[[package]] +name = "tcp-stream" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" +dependencies = [ + "cfg-if", + "p12-keystore", + "rustls-connector", + "rustls-pemfile 2.2.0", +] + [[package]] name = "temp_deno_which" version = "0.1.0" @@ -12145,7 +12723,7 @@ version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "fastrand", + "fastrand 2.5.0", "getrandom 0.4.3", "once_cell", "rustix 1.1.4", @@ -12201,11 +12779,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.19", ] [[package]] @@ -12221,13 +12799,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -12324,9 +12902,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.53" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -12344,9 +12922,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -12455,6 +13033,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-executor-trait" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6278565f9fd60c2d205dfbc827e8bb1236c2b1a57148708e95861eff7a6b3bad" +dependencies = [ + "async-trait", + "executor-trait", + "tokio", +] + [[package]] name = "tokio-graceful" version = "0.1.6" @@ -12514,6 +13103,20 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9481a72f36bd9cbb8d6dd349227c4783e234e4332cfe806225bc929c4b92486" +dependencies = [ + "async-trait", + "futures-core", + "futures-io", + "reactor-trait", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-retry2" version = "0.5.8" @@ -12569,9 +13172,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -12740,7 +13343,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -12772,7 +13375,7 @@ dependencies = [ "http 1.4.2", "http-body 1.1.0", "http-body-util", - "hyper 1.10.1", + "hyper 1.11.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -12852,7 +13455,7 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -12902,7 +13505,7 @@ checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", "symlink", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", "tracing-subscriber", ] @@ -13125,9 +13728,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" [[package]] name = "typed-path" @@ -13149,9 +13752,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typetag" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" dependencies = [ "erased-serde", "inventory", @@ -13162,13 +13765,13 @@ dependencies = [ [[package]] name = "typetag-impl" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -13475,9 +14078,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -13492,7 +14095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" dependencies = [ "bindgen 0.71.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "fslock", "gzip-header", "home", @@ -13544,6 +14147,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "walkdir" version = "2.5.0" @@ -13707,7 +14316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51cf5f08b357e64cd7642ab4bbeb11aecab9e15520692129624fb9908b8df2c" dependencies = [ "deno_error 0.6.1", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -13752,15 +14361,15 @@ checksum = "974fa1e325e6cc5327de8887f189a441fcff4f8eedcd31ec87f0ef0cc5283fbc" dependencies = [ "bytes", "http 1.4.2", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", ] [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" dependencies = [ "rustls-pki-types", ] @@ -13771,14 +14380,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] @@ -13870,7 +14479,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-nats", @@ -13885,6 +14494,8 @@ dependencies = [ "futures", "gethostname", "git-version", + "hex", + "hmac", "lazy_static", "once_cell", "opentelemetry 0.30.0", @@ -13933,6 +14544,7 @@ dependencies = [ "windmill-parser-ts", "windmill-queue", "windmill-runtime-nativets", + "windmill-store", "windmill-test-utils", "windmill-trigger", "windmill-trigger-azure", @@ -13952,7 +14564,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.759.0" +version = "1.770.0" dependencies = [ "async-stream", "async-trait", @@ -13985,7 +14597,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13998,7 +14610,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "argon2", @@ -14028,7 +14640,7 @@ dependencies = [ "hex", "hmac", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "indexmap 2.14.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -14119,6 +14731,7 @@ dependencies = [ "windmill-queue", "windmill-store", "windmill-trigger", + "windmill-trigger-amqp", "windmill-trigger-azure", "windmill-trigger-email", "windmill-trigger-gcp", @@ -14136,12 +14749,12 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "lazy_static", "quick_cache", "serde", @@ -14159,7 +14772,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14174,7 +14787,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14200,7 +14813,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.759.0" +version = "1.770.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14210,7 +14823,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14227,7 +14840,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14249,7 +14862,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14272,7 +14885,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14288,11 +14901,11 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", - "hyper 1.10.1", + "hyper 1.11.0", "serde", "serde_json", "sql-builder", @@ -14309,7 +14922,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14330,7 +14943,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14344,7 +14957,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-nats", @@ -14379,14 +14992,14 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", "base64 0.22.1", "chrono", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "lazy_static", "serde", "serde_json", @@ -14404,7 +15017,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14422,7 +15035,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14444,7 +15057,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14464,13 +15077,13 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "itertools 0.14.0", "lazy_static", "prometheus", @@ -14501,7 +15114,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14529,7 +15142,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.759.0" +version = "1.770.0" dependencies = [ "lazy_static", "serde", @@ -14541,14 +15154,14 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.759.0" +version = "1.770.0" dependencies = [ "argon2", "axum 0.8.9", "chrono", "dashmap", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "lazy_static", "serde", "serde_json", @@ -14566,7 +15179,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14580,14 +15193,14 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.759.0" +version = "1.770.0" dependencies = [ "axum 0.8.9", "chrono", "futures", "hex", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "lazy_static", "magic-crypt", "regex", @@ -14615,7 +15228,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.759.0" +version = "1.770.0" dependencies = [ "chrono", "lazy_static", @@ -14629,7 +15242,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14638,7 +15251,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "uuid", @@ -14648,7 +15261,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.759.0" +version = "1.770.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14665,7 +15278,7 @@ dependencies = [ "axum 0.8.9", "backon", "base64 0.22.1", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "chrono", "chrono-tz", @@ -14685,13 +15298,14 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.10.1", + "hyper 1.11.0", "indexmap 2.14.0", "itertools 0.14.0", "jsonwebtoken 8.3.0", "lazy_static", "magic-crypt", "mail-send", + "memchr", "native-tls", "once_cell", "openidconnect", @@ -14726,7 +15340,7 @@ dependencies = [ "systemstat", "tar", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tikv-jemalloc-ctl", "tokio", "tokio-postgres", @@ -14744,13 +15358,14 @@ dependencies = [ "windmill-parser", "windmill-parser-py", "windmill-parser-sql", + "windmill-parser-sql-asset", "windmill-parser-ts", "windmill-types", ] [[package]] name = "windmill-dep-map" -version = "1.759.0" +version = "1.770.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14769,7 +15384,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.759.0" +version = "1.770.0" dependencies = [ "regex", "serde", @@ -14784,7 +15399,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14808,7 +15423,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "futures", @@ -14825,7 +15440,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.759.0" +version = "1.770.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14841,7 +15456,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -14862,7 +15477,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -14893,7 +15508,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "arc-swap", @@ -14918,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-stream", @@ -14952,7 +15567,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "futures", @@ -14962,7 +15577,7 @@ dependencies = [ "serde_json", "serde_yml", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "windmill-common", @@ -14970,7 +15585,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.759.0" +version = "1.770.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14979,7 +15594,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -14991,7 +15606,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -15003,7 +15618,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "gosyn", @@ -15015,7 +15630,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -15027,7 +15642,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -15039,7 +15654,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "nu-parser", @@ -15050,7 +15665,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15061,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15073,7 +15688,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15084,7 +15699,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-recursion", @@ -15106,7 +15721,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -15118,7 +15733,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -15132,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15149,7 +15764,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -15162,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde", @@ -15174,7 +15789,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -15192,7 +15807,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15208,7 +15823,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15224,7 +15839,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde", @@ -15235,7 +15850,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-recursion", @@ -15262,7 +15877,7 @@ dependencies = [ "serde_urlencoded", "sql-builder", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "ulid", @@ -15274,7 +15889,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "const_format", @@ -15314,7 +15929,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.759.0" +version = "1.770.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15325,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-recursion", @@ -15335,7 +15950,7 @@ dependencies = [ "futures", "hex", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "lazy_static", "magic-crypt", "quick_cache", @@ -15359,7 +15974,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15383,14 +15998,14 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", "axum 0.8.9", "chrono", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "itertools 0.14.0", "lazy_static", "rand 0.9.0", @@ -15414,9 +16029,36 @@ dependencies = [ "windmill-queue", ] +[[package]] +name = "windmill-trigger-amqp" +version = "1.770.0" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.8.9", + "base64 0.22.1", + "futures", + "itertools 0.14.0", + "lapin", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.19", + "tokio", + "tokio-executor-trait", + "tokio-reactor-trait", + "tracing", + "urlencoding", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + [[package]] name = "windmill-trigger-azure" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15436,7 +16078,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tracing", @@ -15449,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15469,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15488,7 +16130,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tonic 0.13.1", @@ -15503,7 +16145,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15515,7 +16157,7 @@ dependencies = [ "hex", "hmac", "http 1.4.2", - "hyper 1.10.1", + "hyper 1.11.0", "itertools 0.14.0", "lazy_static", "matchit 0.7.3", @@ -15526,7 +16168,7 @@ dependencies = [ "sha1", "sha2 0.10.9", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "windmill-api-auth", @@ -15539,7 +16181,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15562,7 +16204,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15574,7 +16216,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "windmill-api-auth", @@ -15586,7 +16228,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-nats", @@ -15610,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15630,7 +16272,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-postgres", "tokio-stream", @@ -15645,7 +16287,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15661,7 +16303,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "windmill-api-auth", @@ -15673,7 +16315,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-trait", @@ -15698,10 +16340,10 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", - "bitflags 2.13.0", + "bitflags 2.13.1", "chrono", "hex", "itertools 0.14.0", @@ -15717,7 +16359,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-once-cell", @@ -15737,6 +16379,7 @@ dependencies = [ "dotenv", "eventsource-stream", "flume", + "fs4", "futures", "gcp_auth", "git-version", @@ -15744,7 +16387,7 @@ dependencies = [ "hmac", "hudsucker", "hyper-http-proxy", - "hyper-tls", + "hyper-rustls 0.27.9", "hyper-util", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15774,6 +16417,9 @@ dependencies = [ "reqwest-middleware", "rsa", "rust_decimal", + "rustls 0.23.35", + "rustls-native-certs 0.8.4", + "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.9", @@ -15783,6 +16429,7 @@ dependencies = [ "tiberius", "tokio", "tokio-postgres", + "tokio-rustls 0.26.4", "tokio-stream", "tokio-util", "tracing", @@ -15821,13 +16468,13 @@ dependencies = [ "windmill-types", "windmill-worker-volumes", "windows 0.61.3", - "x509-parser", + "x509-parser 0.16.0", "yaml-rust", ] [[package]] name = "windmill-worker-volumes" -version = "1.759.0" +version = "1.770.0" dependencies = [ "bytes", "futures", @@ -16078,7 +16725,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "widestring", "windows-sys 0.52.0", ] @@ -16473,24 +17120,52 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", +] + [[package]] name = "x509-parser" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" dependencies = [ - "asn1-rs", + "asn1-rs 0.6.2", "data-encoding", - "der-parser", + "der-parser 9.0.0", "lazy_static", "nom", - "oid-registry", + "oid-registry 0.7.1", "ring 0.17.14", "rusticata-macros", "thiserror 1.0.69", "time", ] +[[package]] +name = "x509-parser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" +dependencies = [ + "asn1-rs 0.7.2", + "data-encoding", + "der-parser 10.0.0", + "lazy_static", + "nom", + "oid-registry 0.8.1", + "rusticata-macros", + "thiserror 2.0.19", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -16565,18 +17240,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index acaef26b20..b10de4017f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.759.0" +version = "1.770.0" authors.workspace = true edition.workspace = true @@ -22,6 +22,7 @@ members = [ "./windmill-trigger-kafka", "./windmill-trigger-postgres", "./windmill-trigger-mqtt", + "./windmill-trigger-amqp", "./windmill-trigger-websocket", "./windmill-trigger-email", "./windmill-trigger-nats", @@ -87,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.759.0" +version = "1.770.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -141,6 +142,7 @@ postgres_trigger = ["windmill-api/postgres_trigger"] mcp = ["windmill-ai/mcp", "windmill-api/mcp", "windmill-worker/mcp"] bedrock = ["windmill-ai/bedrock", "windmill-api/bedrock", "windmill-worker/bedrock"] mqtt_trigger = ["windmill-api/mqtt_trigger"] +amqp_trigger = ["windmill-api/amqp_trigger"] native_trigger = ["windmill-api/native_trigger"] sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"] gcp_trigger = ["windmill-api/gcp_trigger"] @@ -166,21 +168,22 @@ mssql = ["windmill-worker/mssql"] mssql-kerberos = ["windmill-worker/mssql-kerberos"] # Linux/Unix integrated auth mssql-winauth = ["windmill-worker/mssql-winauth"] # Windows integrated auth bigquery = ["windmill-worker/bigquery"] +snowflake = ["windmill-worker/snowflake"] php = ["windmill-worker/php"] csharp = ["windmill-worker/csharp"] nu = ["windmill-worker/nu"] java = ["windmill-worker/java"] ruby = ["windmill-worker/ruby"] rlang = ["windmill-worker/rlang"] -all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "csharp", "nu", "php", "java", "ruby", "rlang"] +all_languages = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-kerberos", "bigquery", "snowflake", "csharp", "nu", "php", "java", "ruby", "rlang"] # For windows we have another set of languages enabled -all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "csharp", "nu", "php", "java"] +all_languages_windows = ["python", "deno_core", "rust", "mysql", "oracledb", "duckdb", "mssql-winauth", "bigquery", "snowflake", "csharp", "nu", "php", "java", "ruby", "rlang"] # Edition meta-features: shared groups run_inline = ["windmill-api/run_inline"] oss_core = [ "embedding", "parquet", "openidconnect", "license", "http_trigger", "zip", "oauth2", "postgres_trigger", - "mqtt_trigger", "websocket", "smtp", "native_trigger", + "mqtt_trigger", "amqp_trigger", "websocket", "smtp", "native_trigger", "static_frontend", "mcp", "bedrock", "run_inline", "quickjs" ] @@ -198,10 +201,15 @@ ce = ["ce_rpi", "jemalloc", "dind", "agent_worker_server"] # Edition meta-features: EE variants ee = ["ce", "ee_core", "ee_server", "kafka-gssapi"] ee_rhel = ["ce_core", "ee_core", "kafka-gssapi", "all_languages"] -ee_windows = ["ce_core", "ee_core", "all_languages_windows"] +# The Windows binary is worker-only, but a non-agent worker runs windmill-api on +# localhost (main.rs run_server, `if !is_agent`) and jobs call back into it, so +# it needs every feature its own plumbing or its jobs invoke in-process; drop +# only external/server-facing surface the worker never runs. +worker_windows_core = ["private", "operator", "parquet", "quickjs", "enterprise", "prometheus", "otel", "jemalloc", "windmill-worker/mcp", "windmill-store/mcp", "windmill-worker/bedrock", "openidconnect", "run_inline", "windmill-api/instance_smtp", "oauth2"] +ee_windows = ["worker_windows_core", "all_languages_windows"] all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embedding", "parquet", "prometheus", "flow_testing", "openidconnect", "cloud", "jemalloc", "tantivy", "sqlx", "kafka", "kafka-gssapi", "nats", "otel", "dind", "websocket", "http_trigger", - "postgres_trigger", "mcp", "mqtt_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", "smtp", "stripe", + "postgres_trigger", "mcp", "mqtt_trigger", "amqp_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", "smtp", "stripe", "license", "oauth2", "zip", "static_frontend", "scoped_cache", "agent_worker_server", "bedrock", "native_trigger", "quickjs", "windmill-git-sync/all_sqlx_features"] @@ -250,6 +258,7 @@ windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-object-store.workspace = true windmill-git-sync.workspace = true +windmill-store.workspace = true windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-scripts.workspace = true @@ -326,6 +335,8 @@ async-nats.workspace = true aws-sdk-sqs.workspace = true aws-config.workspace = true aws-credential-types.workspace = true +hmac.workspace = true +hex.workspace = true [workspace.dependencies] @@ -358,6 +369,7 @@ windmill-trigger = { path = "./windmill-trigger" } windmill-trigger-kafka = { path = "./windmill-trigger-kafka" } windmill-trigger-postgres = { path = "./windmill-trigger-postgres" } windmill-trigger-mqtt = { path = "./windmill-trigger-mqtt" } +windmill-trigger-amqp = { path = "./windmill-trigger-amqp" } windmill-trigger-websocket = { path = "./windmill-trigger-websocket" } windmill-trigger-email = { path = "./windmill-trigger-email" } windmill-trigger-nats = { path = "./windmill-trigger-nats" } @@ -577,6 +589,7 @@ rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} jsonwebtoken = "8.3.0" pem = "3.0.1" nix = { version = "0.27.1", features = ["process", "signal"] } +fs4 = "0.13" tinyvector = { git = "https://github.com/windmill-labs/tinyvector", rev = "20823b94c20f2b9093f318badd24026cf54dcc85" } hf-hub = "0.4.3" tokenizers = "0.14.1" @@ -675,8 +688,14 @@ tree-sitter-ruby = "=0.23.1" tree-sitter-r = "=1.2.0" oracle = { version = "0.6.3", features = ["chrono"] } rumqttc = { version = "0.24.0", features = ["use-native-tls"]} +lapin = "2.5" +tokio-executor-trait = "2.1" +tokio-reactor-trait = "1.1" strum = { version = "0.27", features = ["derive"] } strum_macros = "0.27" hudsucker = { version = "0.22", features = ["rcgen-ca", "native-tls-client"] } -hyper-http-proxy = { version = "1", default-features = false, features = ["native-tls"] } +hyper-http-proxy = { version = "1", default-features = false, features = ["rustls-tls-native-roots"] } +hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "ring", "tls12"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } +rustls-native-certs = "0.8" rcgen = "0.13" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c44448e6ae..411be49883 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -64ddab2ec262873a6f3ffc4e9d26937d2a2f820b \ No newline at end of file +bee688fce40ac92c088d849d6162e489cf0987a3 \ No newline at end of file diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index b120a00d39..20ea1a0a6c 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -165,13 +165,14 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt conflicts = (path_keys & query_keys) | (path_keys & body_keys) | (query_keys & body_keys) - path_field_renames = {} query_field_renames = {} body_field_renames = {} for field in conflicts: + # The path parameter keeps the plain name: it identifies the item, is what a + # caller reaches for first, and matches the non-colliding endpoints (`getFlowByPath` + # takes `path`). Only the other locations carry a suffix. schemas_and_renames = [ - (path_params_schema, path_keys, '__path', path_field_renames), (query_params_schema, query_keys, '__query', query_field_renames), (body_schema, body_keys, '__body', body_field_renames), ] @@ -197,11 +198,26 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt # Store the reverse mapping: renamed -> original renames_map[new_name] = field + # A body field colliding with a same-named path parameter (`path` on the update + # endpoints) holds the new value and differs only when moving the item, so it must + # stay optional; the server defaults it from the URL path when the caller omits it. + if field in path_keys and field in body_keys and body_schema: + body_name = field + '__body' + if 'required' in body_schema: + body_schema['required'] = [r for r in body_schema['required'] if r != body_name] + prop = body_schema['properties'].get(body_name) + if isinstance(prop, dict): + existing_desc = prop.get('description', '').rstrip('. ') + prop['description'] = ( + f"{existing_desc}. Defaults to `{field}` when omitted; " + f"set it only to change the {field}." + ).lstrip('. ') + # Return None for empty schemas path_params_schema = path_params_schema if path_params_schema and path_params_schema.get('properties') else None query_params_schema = query_params_schema if query_params_schema and query_params_schema.get('properties') else None - return (path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames) + return (path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames) # Cache for loaded external files _external_file_cache: Dict[str, Dict[str, Any]] = {} @@ -453,7 +469,7 @@ export const mcpEndpointTools: EndpointTool[] = []; method = tool['method'].upper() # Generate separate schemas - path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames = extract_separate_schemas( + path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames = extract_separate_schemas( tool['parameters'], tool['requestBody'], spec, tool['required_fields'], base_path, tool.get('include_fields'), tool.get('opaque_fields'), tool.get('include_query_params') ) @@ -462,7 +478,6 @@ export const mcpEndpointTools: EndpointTool[] = []; path_params_ts = json.dumps(path_params_schema, indent=8) if path_params_schema else "undefined" query_params_ts = json.dumps(query_params_schema, indent=8) if query_params_schema else "undefined" body_schema_ts = json.dumps(body_schema, indent=8) if body_schema else "undefined" - path_field_renames_ts = json.dumps(path_field_renames, indent=8) if path_field_renames else "undefined" query_field_renames_ts = json.dumps(query_field_renames, indent=8) if query_field_renames else "undefined" body_field_renames_ts = json.dumps(body_field_renames, indent=8) if body_field_renames else "undefined" @@ -476,7 +491,6 @@ export const mcpEndpointTools: EndpointTool[] = []; pathParamsSchema: {path_params_ts}, queryParamsSchema: {query_params_ts}, bodySchema: {body_schema_ts}, - pathFieldRenames: {path_field_renames_ts}, queryFieldRenames: {query_field_renames_ts}, bodyFieldRenames: {body_field_renames_ts} }}""" @@ -497,7 +511,6 @@ export interface EndpointTool {{ pathParamsSchema?: object; queryParamsSchema?: object; bodySchema?: object; - pathFieldRenames?: Record; queryFieldRenames?: Record; bodyFieldRenames?: Record; }} @@ -529,7 +542,7 @@ pub fn all_tools() -> Vec {{ method = tool['method'].upper() # Generate separate schemas - path_params_schema, query_params_schema, body_schema, path_field_renames, query_field_renames, body_field_renames = extract_separate_schemas( + path_params_schema, query_params_schema, body_schema, query_field_renames, body_field_renames = extract_separate_schemas( tool['parameters'], tool['requestBody'], spec, tool['required_fields'], base_path, tool.get('include_fields'), tool.get('opaque_fields'), tool.get('include_query_params') ) @@ -537,7 +550,6 @@ pub fn all_tools() -> Vec {{ path_params_rust = schema_to_rust_value(path_params_schema) query_params_rust = schema_to_rust_value(query_params_schema) body_schema_rust = schema_to_rust_value(body_schema) - path_field_renames_rust = schema_to_rust_value(path_field_renames if path_field_renames else None) query_field_renames_rust = schema_to_rust_value(query_field_renames if query_field_renames else None) body_field_renames_rust = schema_to_rust_value(body_field_renames if body_field_renames else None) @@ -551,7 +563,6 @@ pub fn all_tools() -> Vec {{ path_params_schema: {path_params_rust}, query_params_schema: {query_params_rust}, body_schema: {body_schema_rust}, - path_field_renames: {path_field_renames_rust}, query_field_renames: {query_field_renames_rust}, body_field_renames: {body_field_renames_rust}, }}""" diff --git a/backend/migrations/20260716080248_widen_healthchecks_check_type.down.sql b/backend/migrations/20260716080248_widen_healthchecks_check_type.down.sql new file mode 100644 index 0000000000..86e8d787dc --- /dev/null +++ b/backend/migrations/20260716080248_widen_healthchecks_check_type.down.sql @@ -0,0 +1,2 @@ +-- Rows written since the up migration may exceed 50 chars; truncate so the cast succeeds. +ALTER TABLE healthchecks ALTER COLUMN check_type TYPE varchar(50) USING left(check_type, 50); diff --git a/backend/migrations/20260716080248_widen_healthchecks_check_type.up.sql b/backend/migrations/20260716080248_widen_healthchecks_check_type.up.sql new file mode 100644 index 0000000000..697d5577ab --- /dev/null +++ b/backend/migrations/20260716080248_widen_healthchecks_check_type.up.sql @@ -0,0 +1,4 @@ +-- Alert tags embed unbounded components (mountpoint, hostname), so they overflowed +-- varchar(50). create_alert only logs the insert error while the notification still +-- fires, so an overflowing tag re-alerts every monitor pass and never recovers. +ALTER TABLE healthchecks ALTER COLUMN check_type TYPE text; diff --git a/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.down.sql b/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.down.sql new file mode 100644 index 0000000000..ab9357e09e --- /dev/null +++ b/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.down.sql @@ -0,0 +1,2 @@ +REVOKE ALL ON zombie_job_counter FROM windmill_user; +REVOKE ALL ON zombie_job_counter FROM windmill_admin; diff --git a/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.up.sql b/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.up.sql new file mode 100644 index 0000000000..b8db8dfb0d --- /dev/null +++ b/backend/migrations/20260716151337_grant_zombie_job_counter_to_windmill_roles.up.sql @@ -0,0 +1,14 @@ +-- The zombie_job_counter table (migration 20250205131522) was created relying on +-- ALTER DEFAULT PRIVILEGES to grant access to windmill_user and windmill_admin. +-- Those default privileges only apply to objects created by the role that set +-- them (migration 20250205131523), so deployments whose migration runner is a +-- different role leave zombie_job_counter ungranted. The table used to be +-- reached only through ON DELETE CASCADE, which bypasses caller permissions, +-- but 20260625092813_drop_v2_job_side_table_cascades replaced that cascade with +-- an explicit DELETE in delete_jobs (windmill-common/src/jobs.rs) which runs as +-- the invoking role and fails with "permission denied for table +-- zombie_job_counter". Grant explicitly to guarantee access regardless of who +-- ran the migrations (same fix as notify_event in 20260619091631, +-- script_trigger in 20260619112847 and dispatch_event in 20260701080313). +GRANT ALL ON zombie_job_counter TO windmill_user; +GRANT ALL ON zombie_job_counter TO windmill_admin; diff --git a/backend/migrations/20260716152346_custom_instance_replication_user.down.sql b/backend/migrations/20260716152346_custom_instance_replication_user.down.sql new file mode 100644 index 0000000000..2350875885 --- /dev/null +++ b/backend/migrations/20260716152346_custom_instance_replication_user.down.sql @@ -0,0 +1,14 @@ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN + ALTER ROLE custom_instance_user REPLICATION; + END IF; + + DROP ROLE IF EXISTS custom_instance_replication_user; + + DELETE FROM global_settings WHERE name = 'custom_instance_replication_pwd'; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'custom_instance_replication_user down-migration error, skipping: %', SQLERRM; +END +$$; diff --git a/backend/migrations/20260716152346_custom_instance_replication_user.up.sql b/backend/migrations/20260716152346_custom_instance_replication_user.up.sql new file mode 100644 index 0000000000..3fb0f0d9d0 --- /dev/null +++ b/backend/migrations/20260716152346_custom_instance_replication_user.up.sql @@ -0,0 +1,34 @@ +-- Dedicated logical-replication role used by postgres triggers on custom-instance +-- datatables. Its password is stored server-only in global_settings.custom_instance_replication_pwd +-- (hidden from the config surface); membership in custom_instance_user lets it manage +-- publications on the datatable tables. custom_instance_user itself must not hold REPLICATION. +DO $$ +DECLARE + pwd text; +BEGIN + SELECT gen_random_uuid()::text INTO pwd; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_replication_user') THEN + EXECUTE format('ALTER USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + ELSE + EXECUTE format('CREATE USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN + GRANT custom_instance_user TO custom_instance_replication_user; + ALTER ROLE custom_instance_user NOREPLICATION; + END IF; + + INSERT INTO global_settings (name, value) + VALUES ('custom_instance_replication_pwd', to_jsonb(pwd::text)) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + + -- Drop any replication password an earlier iteration stored in the operator-facing row. + UPDATE global_settings + SET value = value - 'replication_user_pwd' + WHERE name = 'custom_instance_pg_databases'; +EXCEPTION + WHEN others THEN + RAISE NOTICE 'custom_instance_replication_user migration error, skipping: %', SQLERRM; +END +$$; diff --git a/backend/migrations/20260720050649_data_metric_catalog.down.sql b/backend/migrations/20260720050649_data_metric_catalog.down.sql new file mode 100644 index 0000000000..91227d6a13 --- /dev/null +++ b/backend/migrations/20260720050649_data_metric_catalog.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS data_metric; diff --git a/backend/migrations/20260720050649_data_metric_catalog.up.sql b/backend/migrations/20260720050649_data_metric_catalog.up.sql new file mode 100644 index 0000000000..3f701eafca --- /dev/null +++ b/backend/migrations/20260720050649_data_metric_catalog.up.sql @@ -0,0 +1,36 @@ +-- Catalog of `// measure` / `// dimension` declarations, one row per declaration, +-- synced from the producing script's annotations on deploy. Persisted (rather than +-- parsed from script content on demand) so folder-scoped listing is an index range +-- scan instead of a scan over every script body in the folder. +CREATE TABLE data_metric ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON UPDATE CASCADE ON DELETE CASCADE, + -- The producing script. Also the permission anchor: ducklake table paths have + -- no folder to authorize against, so reads are filtered on this. + script_path VARCHAR(510) NOT NULL, + table_path VARCHAR(510) NOT NULL, + kind VARCHAR(16) NOT NULL CHECK (kind IN ('measure', 'dimension')), + name VARCHAR(255) NOT NULL, + expr TEXT NOT NULL, + filter TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path, kind, name) +); + +-- "what does this table declare?" for the script editor drawer. +CREATE INDEX idx_data_metric_table ON data_metric (workspace_id, table_path); + +-- Sort order for the listing endpoint's `ORDER BY table_path, kind, name, +-- script_path`, so each keyset page is an ordered index range scan resuming from +-- the previous page's last row rather than re-sorting the catalog every request. +CREATE INDEX idx_data_metric_page ON data_metric (workspace_id, table_path, kind, name, script_path); + +-- "what is declared under this folder?" for the agent tool. text_pattern_ops so a +-- `LIKE 'f/folder/%'` prefix is a range scan: this database's collation is not C, +-- and under a locale collation the planner will not use a default-opclass index +-- for prefix matching. +CREATE INDEX idx_data_metric_folder ON data_metric (workspace_id, script_path text_pattern_ops); + +-- Written on user_db transactions (SET LOCAL ROLE); the one-time GRANT ALL +-- migration predates this table, so explicit grants are required. +GRANT ALL ON data_metric TO windmill_user; +GRANT ALL ON data_metric TO windmill_admin; diff --git a/backend/migrations/20260720081307_add_feature_usage.down.sql b/backend/migrations/20260720081307_add_feature_usage.down.sql new file mode 100644 index 0000000000..1b12797d72 --- /dev/null +++ b/backend/migrations/20260720081307_add_feature_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE feature_usage; diff --git a/backend/migrations/20260720081307_add_feature_usage.up.sql b/backend/migrations/20260720081307_add_feature_usage.up.sql new file mode 100644 index 0000000000..6d17777b97 --- /dev/null +++ b/backend/migrations/20260720081307_add_feature_usage.up.sql @@ -0,0 +1,17 @@ +-- Generic product-telemetry accumulator: day-bucketed counters (entity_id = '') +-- and per-entity accumulators (e.g. messages per AI session). Aggregated into +-- the anonymous usage stats payload and pruned after 60 days. +CREATE TABLE feature_usage ( + feature VARCHAR(50) NOT NULL, + kind VARCHAR(50) NOT NULL, + key VARCHAR(100) NOT NULL DEFAULT '', + entity_id VARCHAR(50) NOT NULL DEFAULT '', + day DATE NOT NULL DEFAULT CURRENT_DATE, + value BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (feature, kind, key, entity_id, day) +); + +-- The periodic retention delete filters on day alone; without this it would +-- full-scan the table (the PK only reaches day through four other columns). +CREATE INDEX idx_feature_usage_day ON feature_usage (day); diff --git a/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql b/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql new file mode 100644 index 0000000000..f4b3030011 --- /dev/null +++ b/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS ai_chat_usage ( + id BIGSERIAL PRIMARY KEY, + session_id VARCHAR(36) NOT NULL UNIQUE, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + mode VARCHAR(50) NOT NULL, + message_count INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_ai_chat_usage_created_at ON ai_chat_usage (created_at); diff --git a/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql b/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql new file mode 100644 index 0000000000..7c72379b4f --- /dev/null +++ b/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql @@ -0,0 +1,22 @@ +-- AI chat usage telemetry now flows through the generic feature_usage table +-- (ai_chat/message and ai_chat/model events). Backfill the accumulated rows so +-- no reporting window is lost, then drop the old table. Day-bucketing uses the +-- chat's first-message date; values are filtered to the identifier shape the +-- logging endpoint enforces. +INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at) +SELECT 'ai_chat', 'message', mode, session_id, created_at::date, message_count, created_at +FROM ai_chat_usage +WHERE mode ~ '^[A-Za-z0-9_:./-]{1,100}$' + AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$' +ON CONFLICT (feature, kind, key, entity_id, day) +DO UPDATE SET value = feature_usage.value + EXCLUDED.value; + +INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at) +SELECT 'ai_chat', 'model', provider || ':' || model, session_id, created_at::date, message_count, created_at +FROM ai_chat_usage +WHERE (provider || ':' || model) ~ '^[A-Za-z0-9_:./-]{1,100}$' + AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$' +ON CONFLICT (feature, kind, key, entity_id, day) +DO UPDATE SET value = feature_usage.value + EXCLUDED.value; + +DROP TABLE ai_chat_usage; diff --git a/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.down.sql b/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.down.sql new file mode 100644 index 0000000000..0f778d3edf --- /dev/null +++ b/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.down.sql @@ -0,0 +1,3 @@ +-- No down migration: revoking these grants leaves the instance unable to run +-- any query on a user_db transaction. 20250205131523, which this re-runs, is +-- likewise a no-op down. diff --git a/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.up.sql b/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.up.sql new file mode 100644 index 0000000000..33ac63804a --- /dev/null +++ b/backend/migrations/20260720131744_regrant_schema_and_tables_to_windmill_roles.up.sql @@ -0,0 +1,107 @@ +-- Re-runs the grants of 20250205131523. Everything in that migration sits in +-- one DO block whose first statement is LOCK TABLE pg_catalog.pg_roles, which +-- a non-superuser cannot take; on managed Postgres (RDS, Cloud SQL) it raises +-- and the block's single EXCEPTION WHEN OTHERS handler downgrades the failure +-- to a NOTICE, so every GRANT after it is skipped. +-- +-- Most tables survive that anyway, because 20221105003256 grants them outside +-- any such block. What is lost is the ALTER DEFAULT PRIVILEGES, which is what +-- covers tables created by later migrations. Those default privileges only +-- apply to objects created by the role that set them, so a deployment whose +-- migration runner never successfully ran them ends up with newer tables +-- ungranted and writes on a user_db transaction (SET LOCAL ROLE windmill_user / +-- windmill_admin) failing with "permission denied for table". That gap is why +-- 20260619091631, 20260701083047 and 20260716151337 each had to patch one +-- table by hand; re-establishing the default privileges under the current +-- runner closes it for future tables instead. +-- +-- Which relations to grant: only those owned by the migration runner or a role +-- it is an explicit (recursive) member of. That membership set, owner_roles, +-- comes from pg_auth_members, deliberately NOT pg_has_role -- Postgres treats a +-- superuser as a member of every role, so pg_has_role would make a superuser +-- runner grant the windmill roles access to every co-located object in the +-- schema (another application's tables, an extension's tables). Explicit +-- membership still covers the case that owner-name equality would miss: after a +-- migration-credential rotation the objects stay owned by the previous runner +-- while the new runner is a real member of it. +-- +-- This migration must never abort an upgrade, so every grant is guarded +-- individually. That is the opposite of 20250205131523's single block-wide +-- WHEN OTHERS, whose flaw was granularity, not the mere presence of a handler: +-- one early failure there silently skipped every remaining grant. Here each +-- grant that cannot be applied is isolated and re-raised as a named WARNING, so +-- the rest still run and the reason is visible in the migration log. The known +-- failure modes this absorbs: +-- * an object dropped by another session between the catalog scan and the +-- GRANT (only possible when something outside the migration shares the DB). +-- * USAGE on a schema the runner does not own (needed only where USAGE was +-- revoked from PUBLIC, else the roles resolve no table and queries fail +-- with "relation does not exist"); such a deployment needs +-- init-db-as-superuser.sql run by a superuser. +-- windmill_user and windmill_admin are guaranteed to exist: 20221105003256 +-- grants to both outside any handler, so any database that reached this +-- migration already has them. +DO +$do$ +DECLARE + target_schema TEXT := current_schema(); + obj TEXT; + owner_roles OID[]; +BEGIN + WITH RECURSIVE my_roles(oid) AS ( + SELECT oid FROM pg_roles WHERE rolname = current_user + UNION + SELECT m.roleid FROM pg_auth_members m JOIN my_roles r ON m.member = r.oid + ) + SELECT array_agg(oid) INTO owner_roles FROM my_roles; + + BEGIN + EXECUTE format('GRANT USAGE ON SCHEMA %I TO windmill_user, windmill_admin', target_schema); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'skipped GRANT USAGE on schema %: %', target_schema, SQLERRM; + END; + + -- pg_class over the relkinds GRANT ... ON ALL TABLES covers -- ordinary (r), + -- partitioned (p), views (v), materialized views (m), foreign (f). pg_tables + -- would miss views such as flow_workspace_runnables, which are read through + -- user_db transactions and so must be granted too. + FOR obj IN + SELECT format('%I.%I', n.nspname, c.relname) + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = target_schema + AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND c.relowner = ANY(owner_roles) + LOOP + BEGIN + EXECUTE format('GRANT ALL ON TABLE %s TO windmill_user, windmill_admin', obj); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'skipped GRANT on relation %: %', obj, SQLERRM; + END; + END LOOP; + + FOR obj IN + SELECT format('%I.%I', n.nspname, c.relname) + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = target_schema + AND c.relkind = 'S' + AND c.relowner = ANY(owner_roles) + LOOP + BEGIN + EXECUTE format('GRANT ALL ON SEQUENCE %s TO windmill_user, windmill_admin', obj); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'skipped GRANT on sequence %: %', obj, SQLERRM; + END; + END LOOP; + + -- Applies to future objects created by the runner (the FOR ROLE default), + -- so it cannot conflict with objects owned by anyone else. + BEGIN + EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO windmill_user, windmill_admin', target_schema); + EXECUTE format('ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON SEQUENCES TO windmill_user, windmill_admin', target_schema); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'skipped ALTER DEFAULT PRIVILEGES in schema %: %', target_schema, SQLERRM; + END; +END +$do$; diff --git a/backend/migrations/20260721082249_add_amqp_trigger.down.sql b/backend/migrations/20260721082249_add_amqp_trigger.down.sql new file mode 100644 index 0000000000..1ab9baecb8 --- /dev/null +++ b/backend/migrations/20260721082249_add_amqp_trigger.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE amqp_trigger; diff --git a/backend/migrations/20260721082249_add_amqp_trigger.up.sql b/backend/migrations/20260721082249_add_amqp_trigger.up.sql new file mode 100644 index 0000000000..2f2da55c00 --- /dev/null +++ b/backend/migrations/20260721082249_add_amqp_trigger.up.sql @@ -0,0 +1,83 @@ +-- Add up migration script here +CREATE TABLE amqp_trigger ( + amqp_resource_path VARCHAR(255) NOT NULL, + queue_name VARCHAR(255) NOT NULL, + exchange JSONB NULL, + options JSONB NULL, + path VARCHAR(255) NOT NULL, + script_path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + edited_by VARCHAR(50) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + extra_perms JSONB NOT NULL DEFAULT '{}', + server_id VARCHAR(50) NULL, + last_server_ping TIMESTAMPTZ NULL, + error TEXT NULL, + error_handler_path VARCHAR(255) NULL, + error_handler_args JSONB NULL, + retry JSONB NULL, + mode TRIGGER_MODE NOT NULL DEFAULT 'enabled'::TRIGGER_MODE, + permissioned_as VARCHAR(255) NOT NULL, + labels TEXT[] NULL, + PRIMARY KEY (path, workspace_id), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + +CREATE INDEX idx_amqp_trigger_labels ON amqp_trigger USING gin (labels) WHERE labels IS NOT NULL; + +GRANT ALL ON amqp_trigger TO windmill_user; +GRANT ALL ON amqp_trigger TO windmill_admin; + +ALTER TABLE amqp_trigger ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON amqp_trigger FOR ALL TO windmill_admin USING (true); + +CREATE POLICY see_folder_extra_perms_user_select ON amqp_trigger FOR SELECT TO windmill_user +USING (SPLIT_PART(amqp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(amqp_trigger.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_read'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_insert ON amqp_trigger FOR INSERT TO windmill_user +WITH CHECK (SPLIT_PART(amqp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(amqp_trigger.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON amqp_trigger FOR UPDATE TO windmill_user +USING (SPLIT_PART(amqp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(amqp_trigger.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); +CREATE POLICY see_folder_extra_perms_user_delete ON amqp_trigger FOR DELETE TO windmill_user +USING (SPLIT_PART(amqp_trigger.path, '/', 1) = 'f' AND SPLIT_PART(amqp_trigger.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.folders_write'), ','))::text[])); + +CREATE POLICY see_own ON amqp_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(amqp_trigger.path, '/', 1) = 'u' AND SPLIT_PART(amqp_trigger.path, '/', 2) = (select current_setting('session.user'))); +CREATE POLICY see_member ON amqp_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(amqp_trigger.path, '/', 1) = 'g' AND SPLIT_PART(amqp_trigger.path, '/', 2) = any((select regexp_split_to_array(current_setting('session.groups'), ','))::text[])); + +CREATE POLICY see_extra_perms_user_select ON amqp_trigger FOR SELECT TO windmill_user +USING (extra_perms ? (select concat('u/', current_setting('session.user')))); +CREATE POLICY see_extra_perms_user_insert ON amqp_trigger FOR INSERT TO windmill_user +WITH CHECK ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_update ON amqp_trigger FOR UPDATE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); +CREATE POLICY see_extra_perms_user_delete ON amqp_trigger FOR DELETE TO windmill_user +USING ((extra_perms ->> (select concat('u/', current_setting('session.user'))))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON amqp_trigger FOR SELECT TO windmill_user +USING (extra_perms ?| (select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]); +CREATE POLICY see_extra_perms_groups_insert ON amqp_trigger FOR INSERT TO windmill_user +WITH CHECK (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_update ON amqp_trigger FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_delete ON amqp_trigger FOR DELETE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY((select regexp_split_to_array(current_setting('session.pgroups'), ','))::text[]) + AND value::boolean)); + +-- Enum values for the new trigger kind. ALTER TYPE ... ADD VALUE runs inside the +-- migration transaction on PG >= 14 (Windmill's minimum) as long as the value +-- isn't used in the same transaction — the amqp_trigger table above does not +-- reference these enum types. +ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'amqp'; +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'amqp'; +ALTER TYPE draft_kind ADD VALUE IF NOT EXISTS 'trigger_amqp'; diff --git a/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql new file mode 100644 index 0000000000..9cf6c8c064 --- /dev/null +++ b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data repair: once the leading slash is restored, the rows are +-- indistinguishable from paths that always had it. Intentionally a no-op. +SELECT 1; diff --git a/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql new file mode 100644 index 0000000000..cbf0382fcc --- /dev/null +++ b/backend/migrations/20260721154835_fix_s3_asset_paths_missing_leading_slash.up.sql @@ -0,0 +1,144 @@ +-- Repair s3object asset paths recorded without their default-storage leading +-- slash. An S3 asset path is `/` with an empty storage segment +-- (leading slash) for the workspace default: `s3:///exports/x` -> `/exports/x`. +-- Between 2026-07-06 (#9939) and the parser fix, the asset parser stripped +-- leading slashes, so default-storage assets were recorded as `exports/x` — +-- indistinguishable from a secondary storage named `exports`. For rows created +-- in that window (cutoff one day early for safety), a slashless path whose +-- first segment is NOT a storage name — neither a configured secondary storage +-- nor the reserved `_default_` alias — can only be a default-storage key, so it +-- gets its slash back. Rows already starting with `/` are always correct. +-- +-- Best-effort by nature: the corruption itself conflated a stripped default key +-- with a named ref, so identity is inferred from the storage config AS IT IS NOW. +-- A named ref to a storage that was since removed/renamed (or never configured) +-- is the one residual false-positive — it would be repaired as if default. The +-- `created_at` window bounds this for `asset`; `script_trigger` has no timestamp +-- and relies on the storage-name heuristic alone. Both are acceptable given how +-- rare mid-window storage churn is versus the common default-key case this fixes. +-- +-- Same corruption hit `script_trigger.trigger_ref` (the pipeline cascade edges, +-- stored as `s3://`): a default-storage edge recorded as `s3://exports/x` +-- instead of `s3:///exports/x` no longer matches the producer's post-fix write +-- ref at dispatch (asset_dispatch rebuilds `s3://` + the repaired asset path and +-- does an exact `trigger_ref =` match), silently breaking the edge. Repaired with +-- the same storage-name heuristic — script_trigger has no created_at, but a +-- correct default ref is always `s3:///…` and a correct named ref always leads +-- with a real storage name, so a `s3:///…` ref whose seg isn't a storage is +-- unambiguously a slash-stripped default-storage ref. +-- +-- `join_pending_inputs.trigger_ref` (the AND-join barrier) is deliberately NOT +-- repaired: it is transient slot state cleared on fire, so a window-era `s3://…` +-- slot is superseded once inputs re-arrive under the corrected ref (and deleting +-- live slots could drop an in-flight accumulation). materialized_asset_schema is +-- unaffected — it only ever holds ducklake asset_kind, never s3object. +-- +-- Data-repair only: wrapped so a failure NOTICEs and never blocks the release. +DO $migration$ +BEGIN + CREATE TEMP TABLE __asset_slash_fix_cache ( + workspace_id TEXT PRIMARY KEY, + names TEXT[] NOT NULL + ) ON COMMIT DROP; + + -- Reserved first-path-segments that denote a real storage (so a slashless + -- path leading with one is a genuine named ref, NOT a slash-stripped default + -- key): the workspace's secondary_storage names PLUS `_default_`, the alias + -- the runtime treats as the primary storage (workspaces.rs fork_storage_ref). + -- `s3://_default_/key` is a valid explicit-default ref recorded verbatim as + -- `_default_/key`; prepending a slash would corrupt it to key `_default_/key`. + -- The JSON is parsed at most once per workspace (candidate assets can repeat + -- a workspace millions of times via job usages), and only workspaces that + -- actually have candidate rows are ever fetched. + CREATE FUNCTION pg_temp.__asset_slash_fix_storages(ws TEXT) RETURNS TEXT[] AS $fn$ + DECLARE + result TEXT[]; + BEGIN + SELECT c.names INTO result FROM __asset_slash_fix_cache c WHERE c.workspace_id = ws; + IF FOUND THEN + RETURN result; + END IF; + SELECT ARRAY['_default_'] || COALESCE(array_agg(k), '{}') INTO result + FROM workspace_settings s + CROSS JOIN LATERAL jsonb_object_keys( + CASE WHEN jsonb_typeof(s.large_file_storage -> 'secondary_storage') = 'object' + THEN s.large_file_storage -> 'secondary_storage' + ELSE '{}'::JSONB END + ) k + WHERE s.workspace_id = ws; + result := COALESCE(result, ARRAY['_default_']); + INSERT INTO __asset_slash_fix_cache VALUES (ws, result); + RETURN result; + END + $fn$ LANGUAGE plpgsql; + + -- Duplicates first: when the corrected `/path` row already exists for the + -- same usage (recorded before the regression, or re-recorded after the + -- parser fix), prepending the slash would violate the primary key + -- (workspace_id, path, kind, usage_path, usage_kind) — drop the slashless + -- duplicate instead. + DELETE FROM asset a + WHERE a.kind = 's3object' + AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ + AND a.path NOT LIKE '/%' + AND a.path <> '' + AND length(a.path) < 255 + AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)) + AND EXISTS ( + SELECT 1 FROM asset b + WHERE b.workspace_id = a.workspace_id + AND b.path = '/' || a.path + AND b.kind = a.kind + AND b.usage_path = a.usage_path + AND b.usage_kind = a.usage_kind + ); + + -- length < 255 keeps the prepend within the VARCHAR(255) column; a 255-char + -- slashless path cannot be repaired and is left as-is rather than erroring. + UPDATE asset a + SET path = '/' || a.path + WHERE a.kind = 's3object' + AND a.created_at > '2026-07-05 00:00:00+00'::TIMESTAMPTZ + AND a.path NOT LIKE '/%' + AND a.path <> '' + AND length(a.path) < 255 + AND split_part(a.path, '/', 1) <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)); + + -- script_trigger.trigger_ref for asset edges is `s3://`. A corrupted + -- default-storage edge reads `s3:///…` (exactly two slashes); a correct + -- default ref is `s3:///…` and is excluded by the NOT LIKE. `substring(from 6)` + -- is the `` after the `s3://` prefix. Delete a slashless edge whose + -- corrected twin already exists for the same runnable (fetch_subscribers has + -- no DISTINCT, so a duplicate would double-dispatch the subscriber). + DELETE FROM script_trigger a + WHERE a.trigger_kind = 'asset' + AND a.trigger_ref LIKE 's3://%' + AND a.trigger_ref NOT LIKE 's3:///%' + AND substring(a.trigger_ref FROM 6) <> '' + AND split_part(substring(a.trigger_ref FROM 6), '/', 1) + <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)) + AND EXISTS ( + SELECT 1 FROM script_trigger b + WHERE b.workspace_id = a.workspace_id + AND b.runnable_kind = a.runnable_kind + AND b.runnable_path = a.runnable_path + AND b.trigger_kind = a.trigger_kind + AND b.trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6) + ); + + UPDATE script_trigger a + SET trigger_ref = 's3:///' || substring(a.trigger_ref FROM 6) + WHERE a.trigger_kind = 'asset' + AND a.trigger_ref LIKE 's3://%' + AND a.trigger_ref NOT LIKE 's3:///%' + AND substring(a.trigger_ref FROM 6) <> '' + AND split_part(substring(a.trigger_ref FROM 6), '/', 1) + <> ALL (pg_temp.__asset_slash_fix_storages(a.workspace_id)); + + -- The temp table is ON COMMIT DROP; drop the function too so nothing + -- lingers on a pooled connection. + DROP FUNCTION pg_temp.__asset_slash_fix_storages(TEXT); +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'skipping s3 asset leading-slash repair: %', SQLERRM; +END +$migration$; diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql new file mode 100644 index 0000000000..a1e5abccc3 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS error_handler_fallback_to_instance_alerts; diff --git a/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql new file mode 100644 index 0000000000..2ab4b8b673 --- /dev/null +++ b/backend/migrations/20260723215437_error_handler_fallback_to_instance_alerts.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE workspace_settings + ADD COLUMN IF NOT EXISTS error_handler_fallback_to_instance_alerts BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/migrations/20260724094737_runnables_sort_indexes.down.sql b/backend/migrations/20260724094737_runnables_sort_indexes.down.sql new file mode 100644 index 0000000000..a3821b17ab --- /dev/null +++ b/backend/migrations/20260724094737_runnables_sort_indexes.down.sql @@ -0,0 +1,5 @@ +DROP INDEX IF EXISTS index_script_on_workspace_created_at; +DROP INDEX IF EXISTS index_flow_on_workspace_edited_at; +DROP INDEX IF EXISTS index_script_on_workspace_name; +DROP INDEX IF EXISTS index_flow_on_workspace_name; +DROP INDEX IF EXISTS index_app_on_workspace_name; diff --git a/backend/migrations/20260724094737_runnables_sort_indexes.up.sql b/backend/migrations/20260724094737_runnables_sort_indexes.up.sql new file mode 100644 index 0000000000..a1839906bb --- /dev/null +++ b/backend/migrations/20260724094737_runnables_sort_indexes.up.sql @@ -0,0 +1,26 @@ +-- Indexes backing the unified homepage runnables listing so the merged, +-- keyset-paginated query stays an index scan (Merge Append + LIMIT) even on big +-- workspaces, for both the time orders and the name orders. +-- +-- `archived` is the second key so the default (archived = false) and the +-- "Only archived" views each seek their own slice and still get the sort key +-- ordered within it, instead of one view scanning past the other's rows. Apps +-- have no archived column and are excluded from the archived view. + +-- Time orders. +CREATE INDEX IF NOT EXISTS index_script_on_workspace_created_at + ON script (workspace_id, archived, created_at DESC); + +CREATE INDEX IF NOT EXISTS index_flow_on_workspace_edited_at + ON flow (workspace_id, archived, edited_at DESC); + +-- Name orders sort on the lowered summary-or-path expression, so a matching +-- expression index makes that order presorted instead of a full sort per page. +CREATE INDEX IF NOT EXISTS index_script_on_workspace_name + ON script (workspace_id, archived, lower(COALESCE(NULLIF(summary, ''), path))); + +CREATE INDEX IF NOT EXISTS index_flow_on_workspace_name + ON flow (workspace_id, archived, lower(COALESCE(NULLIF(summary, ''), path))); + +CREATE INDEX IF NOT EXISTS index_app_on_workspace_name + ON app (workspace_id, lower(COALESCE(NULLIF(summary, ''), path))); diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index 0f340943aa..2a2d8c310d 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -25,17 +25,23 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { .params .list .iter() - .map(|param| { + .flat_map(|param| { let (otyp, typ) = parse_go_typ(¶m.typ); - Arg { - name: get_name(param), - otyp, - typ, + // a single field can declare several like-typed params: `func main(a, b string)` + let names: Vec = if param.name.is_empty() { + vec!["".to_string()] + } else { + param.name.iter().map(|y| y.name.to_string()).collect() + }; + names.into_iter().map(move |name| Arg { + name, + otyp: otyp.clone(), + typ: typ.clone(), default: None, has_default: false, oidx: None, otyp_inferred: false, - } + }) }) .collect_vec(); Ok(MainArgSignature { @@ -267,6 +273,57 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam Ok(()) } + + #[test] + fn test_parse_go_sig_grouped_params() -> anyhow::Result<()> { + let code = r#" +package main + +func main(a, b string, c int) { +} +"#; + assert_eq!( + parse_go_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: Some("string".to_string()), + name: "a".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + Arg { + otyp: Some("string".to_string()), + name: "b".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + Arg { + otyp: Some("int".to_string()), + name: "c".to_string(), + typ: Typ::Int, + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + }, + ], + auto_kind: None, + has_preprocessor: None, + ..Default::default() + } + ); + + Ok(()) + } } #[test] diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 8d2e87a53e..b8a12fb476 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -383,7 +383,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "test.csv".to_string(), + path: "/test.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -441,7 +441,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "analytics/x.csv".to_string(), + path: "/analytics/x.csv".to_string(), access_type: Some(W), columns: None, },]) @@ -450,10 +450,11 @@ def main(): #[test] fn test_py_write_key_matches_duckdb_read_key() { - // Cross-language lineage: this write records `exports/x`, the same path a - // DuckDB `read_csv('s3://exports/x')` resolves to (see - // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), - // so the producer and consumer connect in the pipeline graph. + // Cross-language lineage: this default-storage write records + // `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')` + // resolves to (see windmill-parser-sql-asset + // `test_duckdb_read_key_matches_sdk_write_key`), so the producer and + // consumer connect in the pipeline graph. let input = r#" import wmill from wmill import S3Object @@ -465,7 +466,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "exports/x".to_string(), + path: "/exports/x".to_string(), access_type: Some(W), columns: None, },]) @@ -508,7 +509,7 @@ def main(): s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "dir/in.csv".to_string(), + path: "/dir/in.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -531,14 +532,14 @@ def main(): Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "mybucket/dir/in.csv".to_string(), - access_type: Some(R), + path: "/out.json".to_string(), + access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.json".to_string(), - access_type: Some(W), + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), columns: None, }, ]) @@ -564,25 +565,25 @@ def main(): Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/enriched.json".to_string(), + path: "/pipelines/km_real/enriched.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/report.json".to_string(), + path: "/pipelines/km_real/report.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/summary.json".to_string(), + path: "/pipelines/km_real/summary.json".to_string(), access_type: Some(W), columns: None, }, diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index d2ab5b51a7..12d7932a16 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -730,6 +730,122 @@ impl Visitor for ColumnIdentCollector { } } +/// Extract the column names a bare SQL expression reads, for validating a +/// metric measure/dimension body against a captured schema. Parses `expr_sql` as +/// a standalone expression (so its identifiers sit at query depth 0, unlike a +/// wrapped `SELECT`) and returns the final segment of each identifier ref: +/// `sum(amount)` → `["amount"]`, `date_trunc('month', ordered_at)` → +/// `["ordered_at"]`, `t.amount` → `["amount"]`, `count(*)` → `[]`. A parse +/// failure yields an empty vec (fail-safe: nothing to check, not a spurious +/// error), matching the fail-soft stance of the rest of the contract machinery. +pub fn extract_expr_column_idents(expr_sql: &str) -> Vec { + let Ok(mut parser) = Parser::new(&DuckDbDialect).try_with_sql(expr_sql) else { + return Vec::new(); + }; + let Ok(expr) = parser.parse_expr() else { + return Vec::new(); + }; + let mut collector = ColumnIdentCollector { refs: Vec::new(), query_depth: 0 }; + let _ = expr.visit(&mut collector); + collector + .refs + .into_iter() + .filter_map(|parts| parts.last().cloned()) + .collect() +} + +/// Whether `expr_sql` is exactly one SQL expression, consuming all of its input. +/// +/// Metric declarations are author text interpolated verbatim into executable SQL +/// that a *reader* then runs. Without this, `count(*) FROM t; DELETE FROM x; SELECT 1` +/// would be stored as a "measure" and execute as whoever opened the drawer, so a +/// declaration that does not parse as a single trailing-token-free expression is +/// rejected at deploy rather than filtered at render time. +pub fn is_single_sql_expression(expr_sql: &str) -> bool { + use sqlparser::tokenizer::{Token, Tokenizer}; + // Reject comment tokens outright: the parser skips them, so `sum(x) --rest` + // would pass the EOF check below, and a comment interpolated into the composed + // SQL can blank the rest of its line (defense in depth — it cannot inject a + // second statement, but a declaration has no business carrying a comment). + match Tokenizer::new(&DuckDbDialect, expr_sql).tokenize() { + Ok(tokens) => { + if tokens.iter().any(|t| { + matches!(t, Token::Whitespace(w) + if matches!(w, sqlparser::tokenizer::Whitespace::SingleLineComment { .. } + | sqlparser::tokenizer::Whitespace::MultiLineComment(_))) + }) { + return false; + } + } + Err(_) => return false, + } + let Ok(mut parser) = Parser::new(&DuckDbDialect).try_with_sql(expr_sql) else { + return false; + }; + if parser.parse_expr().is_err() { + return false; + } + // Anything left over means the expression ended early and the rest would ride + // along into the generated statement. + matches!(parser.peek_token().token, Token::EOF) +} + +struct FunctionCallFinder { + found: bool, +} + +impl Visitor for FunctionCallFinder { + type Break = (); + fn pre_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow<()> { + if matches!(expr, Expr::Function(_)) { + self.found = true; + return std::ops::ControlFlow::Break(()); + } + std::ops::ControlFlow::Continue(()) + } +} + +/// Whether `expr_sql` could be an aggregation. Every SQL aggregate is a function +/// call (`sum(x)`, `count(*)`, a scalar subquery wrapping one, a user-defined +/// aggregate), so an expression with *no* function call anywhere provably cannot +/// aggregate: `revenue = amount` or `amount * 2` is a row-level value, not a +/// measure. Deliberately lenient — any function call earns the benefit of the +/// doubt — to avoid maintaining a reserved aggregate-name list, which drifts with +/// DuckDB versions. A parse failure returns `true` (not our error to raise here; +/// `is_single_sql_expression` already rejects unparseable declarations). +pub fn measure_expr_may_aggregate(expr_sql: &str) -> bool { + let Ok(mut parser) = Parser::new(&DuckDbDialect).try_with_sql(expr_sql) else { + return true; + }; + let Ok(expr) = parser.parse_expr() else { + return true; + }; + let mut finder = FunctionCallFinder { found: false }; + let _ = expr.visit(&mut finder); + finder.found +} + +/// Whether `expr_sql` is exactly one top-level function call, e.g. `sum(amount)` +/// or `count(*)` — but not `sum(a) / count(b)` or `sum(a) + 1`. +/// +/// A `// measure … where ` compiles to ` FILTER (WHERE )`, and +/// SQL binds `FILTER` to the single immediately-preceding aggregate call. If the +/// expression is a composite of several aggregates, the filter silently applies to +/// only the last one and the canonical number is wrong. Requiring one call makes +/// the target of `FILTER` unambiguous. (A single *non-aggregate* call still errors +/// loudly at run time, which is acceptable — the danger is the silent case.) +pub fn is_single_function_call(expr_sql: &str) -> bool { + let Ok(mut parser) = Parser::new(&DuckDbDialect).try_with_sql(expr_sql) else { + return false; + }; + match parser.parse_expr() { + Ok(Expr::Function(_)) => { + matches!(parser.peek_token().token, sqlparser::tokenizer::Token::EOF) + } + _ => false, + } +} + impl Visitor for AssetCollector { type Break = (); @@ -1145,13 +1261,13 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "a.parquet".to_string(), + path: "/a.parquet".to_string(), access_type: Some(R), columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "c.parquet".to_string(), + path: "/c.parquet".to_string(), access_type: Some(W), columns: None }, @@ -1168,25 +1284,35 @@ mod tests { #[test] fn test_duckdb_read_key_matches_sdk_write_key() { // Cross-language lineage: a TS `writeS3File({ s3: "exports/x" })` or - // Python `write_s3_file(S3Object(s3="exports/x"))` records the asset path - // `exports/x` (default storage). A DuckDB reader of the same object must - // resolve to the identical path so the graph connects the producer and - // consumer — both the bare `s3://exports/x` and the triple-slash - // `s3:///exports/x` default-storage form must yield `exports/x`. - for uri in ["s3://exports/x", "s3:///exports/x"] { - let input = format!("SELECT * FROM read_csv('{uri}');"); - let assets = parse_assets(&input).expect("parse").assets; - assert_eq!( - assets, - vec![ParseAssetsResult { - kind: AssetKind::S3Object, - path: "exports/x".to_string(), - access_type: Some(R), - columns: None - }], - "DuckDB read of {uri} must resolve to the SDK write key" - ); - } + // Python `write_s3_file(S3Object(s3="exports/x"))` records the asset + // path `/exports/x` (default storage, leading slash). A DuckDB reader + // of the same object uses the triple-slash default-storage URI and + // must resolve to the identical path so the graph connects producer + // and consumer. The bare `s3://exports/x` form names storage + // `exports` instead — a different object, a different path. + let input = "SELECT * FROM read_csv('s3:///exports/x');"; + let assets = parse_assets(input).expect("parse").assets; + assert_eq!( + assets, + vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/exports/x".to_string(), + access_type: Some(R), + columns: None + }], + ); + + let input = "SELECT * FROM read_csv('s3://exports/x');"; + let assets = parse_assets(input).expect("parse").assets; + assert_eq!( + assets, + vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "exports/x".to_string(), + access_type: Some(R), + columns: None + }], + ); } #[test] @@ -1202,7 +1328,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.csv".to_string(), + path: "/out.csv".to_string(), access_type: Some(W), columns: None }]) @@ -1219,7 +1345,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "referenced.csv".to_string(), + path: "/referenced.csv".to_string(), access_type: Some(R), columns: None }]) @@ -1239,7 +1365,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.csv".to_string(), + path: "/data.csv".to_string(), access_type: Some(RW), columns: None }]) @@ -1259,7 +1385,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.parquet".to_string(), + path: "/data.parquet".to_string(), access_type: Some(RW), columns: None }]) @@ -1277,13 +1403,13 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "a.parquet".to_string(), + path: "/a.parquet".to_string(), access_type: Some(R), columns: None }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "b.parquet".to_string(), + path: "/b.parquet".to_string(), access_type: Some(R), columns: None } @@ -1303,7 +1429,7 @@ mod tests { s.map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "data.parquet".to_string(), + path: "/data.parquet".to_string(), access_type: Some(RW), columns: None }]) @@ -1985,7 +2111,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); @@ -2016,7 +2142,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); assert!(result[0].columns.is_none()); } @@ -2029,7 +2155,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "example_file.parquet"); + assert_eq!(result[0].path, "/example_file.parquet"); let columns = result[0].columns.as_ref().expect("Should have columns"); assert_eq!(columns.get("col1"), Some(&R)); @@ -2048,7 +2174,7 @@ mod tests { assert_eq!(result.len(), 2); assert!(result.iter().any(|a| { - a.path == "file1.parquet" + a.path == "/file1.parquet" && a.columns.as_ref().map_or(false, |c| c.contains_key("col1")) })); assert!(result.iter().any(|a| { @@ -2081,7 +2207,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!(result[0].kind, AssetKind::S3Object); - assert_eq!(result[0].path, "test.parquet"); + assert_eq!(result[0].path, "/test.parquet"); assert_eq!(result[0].access_type, Some(R)); let columns = result[0].columns.as_ref().expect("Should have columns"); diff --git a/backend/parsers/windmill-parser-sql-asset/src/lib.rs b/backend/parsers/windmill-parser-sql-asset/src/lib.rs index 65d8242a80..667cb16b38 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/lib.rs @@ -1,4 +1,7 @@ mod asset_parser; mod asset_parser_utils; -pub use asset_parser::parse_assets; +pub use asset_parser::{ + extract_expr_column_idents, is_single_function_call, is_single_sql_expression, + measure_expr_may_aggregate, parse_assets, +}; pub use asset_parser_utils::parse_wmill_sdk_sql_assets; diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index dc094a0a7b..38f4b7a33e 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -613,19 +613,21 @@ fn parse_pg_file(code: &str) -> anyhow::Result, bool)>> { // the parser's "text" fallback, so the executor can later distinguish // "user committed to text" from "no info, use a placeholder". let mut hm: HashMap = HashMap::new(); - for cap in RE_CODE_PGSQL.captures_iter(code) { - let idx = cap - .get(1) - .and_then(|x| x.as_str().parse::().ok()) - .ok_or_else(|| anyhow!("Impossible to parse arg digit"))?; - + // Walk placeholders with the same tokenizer the executor uses so `$N` + // inside comments, string literals, or dollar-quoted blocks (e.g. the + // commented-out examples in the default postgres template) doesn't + // produce a spurious arg. `range` covers the digits of a real `$N`, so + // anchoring the regex at the preceding `$` only extracts the `::TYPE` cast. + for (idx, range) in parse_pg_statement_arg_positions(code) { // Skip if this arg was explicitly typed in declaration if explicitly_typed_args.contains(&idx) { continue; } - let cast = cap - .get(2) + let cast = RE_CODE_PGSQL + .captures_at(code, range.start - 1) + .filter(|c| c.get(0).is_some_and(|m| m.start() == range.start - 1)) + .and_then(|c| c.get(2)) .map(|cap| transform_types_with_spaces(&cap, &code)); let inferred_default = cast.is_none(); let typ: std::borrow::Cow = cast.unwrap_or(std::borrow::Cow::Borrowed("text")); @@ -1109,6 +1111,31 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT Ok(()) } + #[test] + fn test_parse_pgsql_sig_ignores_placeholders_in_comments_and_strings() -> anyhow::Result<()> { + // Mirrors the default postgres template: the commented-out s3object + // example mentions `$5`, which must not surface as an argument. + let code = r#"-- result_collection=last_statement_all_rows +-- to feed an S3Object (json/jsonl/parquet/csv) as a parameter, declare it as (s3object): +-- -- $5 input_file (s3object) +-- INSERT INTO demo SELECT * FROM jsonb_to_recordset($5::jsonb) AS x(id INT, name TEXT); +-- $1 name1 = default arg +-- $2 name2 +INSERT INTO demo VALUES ($1::TEXT, $2::INT) RETURNING *; +/* also not an arg: $6::int */ +SELECT 'literal $7', "col $8" FROM demo; +"#; + let sig = parse_pgsql_sig(code)?; + assert_eq!( + sig.args + .iter() + .map(|a| (a.oidx, a.name.as_str())) + .collect::>(), + vec![(Some(1), "name1"), (Some(2), "name2")] + ); + Ok(()) + } + #[test] fn test_parse_pgsql_mutli_sig() -> anyhow::Result<()> { let code = r#" diff --git a/backend/parsers/windmill-parser-ts-asset/src/lib.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs index f273db5cbe..68b1287aa0 100644 --- a/backend/parsers/windmill-parser-ts-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -433,7 +433,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "test.csv".to_string(), + path: "/test.csv".to_string(), access_type: Some(R), columns: None, },]) @@ -461,7 +461,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, },]) @@ -470,10 +470,11 @@ mod tests { #[test] fn test_ts_write_key_matches_duckdb_read_key() { - // Cross-language lineage: this write records `exports/x`, the same path a - // DuckDB `read_csv('s3://exports/x')` resolves to (see - // windmill-parser-sql-asset `test_duckdb_read_key_matches_sdk_write_key`), - // so the producer and consumer connect in the pipeline graph. + // Cross-language lineage: this default-storage write records + // `/exports/x`, the same path a DuckDB `read_csv('s3:///exports/x')` + // resolves to (see windmill-parser-sql-asset + // `test_duckdb_read_key_matches_sdk_write_key`), so the producer and + // consumer connect in the pipeline graph. let input = r#" import * as wmill from "windmill-client" export async function main() { @@ -485,7 +486,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "exports/x".to_string(), + path: "/exports/x".to_string(), access_type: Some(W), columns: None, },]) @@ -570,25 +571,25 @@ mod tests { Ok(vec![ ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/enriched.json".to_string(), + path: "/pipelines/km_real/enriched.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/raw_events.json".to_string(), + path: "/pipelines/km_real/raw_events.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/report.json".to_string(), + path: "/pipelines/km_real/report.json".to_string(), access_type: Some(W), columns: None, }, ParseAssetsResult { kind: AssetKind::S3Object, - path: "pipelines/km_real/summary.json".to_string(), + path: "/pipelines/km_real/summary.json".to_string(), access_type: Some(W), columns: None, }, @@ -609,7 +610,7 @@ mod tests { s.map(|r| r.assets).map_err(|e| e.to_string()), Ok(vec![ParseAssetsResult { kind: AssetKind::S3Object, - path: "out.json".to_string(), + path: "/out.json".to_string(), access_type: Some(W), columns: None, },]) diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 222292b087..8a0052d2c7 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.759.0" +version = "1.770.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.759.0" +version = "1.770.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.759.0" +version = "1.770.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.759.0" +version = "1.770.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 4a2ad50f4c..7c866fbd30 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.759.0" +version = "1.770.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index bdce9ec764..de35ccdd6e 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -124,6 +124,15 @@ pub struct ParseAssetsOutput { // column-lineage graph view, executes nothing. #[serde(skip_serializing_if = "Vec::is_empty", default)] pub column_lineage: Vec, + // `// measure = [where ]` — table-scoped aggregations of + // the produced asset. Accumulating, deduped by name. Catalogued at deploy; + // executes nothing. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub measures: Vec, + // `// dimension = ` — table-scoped slicers any measure can be + // grouped by. Accumulating, deduped by name. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub dimensions: Vec, // Bare `// macros` (must be alone on the line, like `// pipeline`) — // marks this DuckDB script as a workspace *macro library*: its body is // CREATE [OR REPLACE] MACRO statements (plus plain setup) registered at @@ -163,6 +172,7 @@ pub enum TriggerSpec { Email, Kafka, Mqtt, + Amqp, Nats, Postgres, Sqs, @@ -493,6 +503,34 @@ pub struct ColumnRef { pub from_column: String, } +// The two table-scoped metric primitives, declared on the script that +// materializes the table (`docs/pipeline-metrics-layer.md`). A *measure* is an +// aggregation; a *dimension* is a slicer. Table-scoped, not per-measure: a +// dimension belongs to the table, so every measure can be sliced by every +// dimension. Metadata only: these are catalogued at deploy and read back by +// editors and agents, which write their own SQL. Names are validated at deploy +// against the producer's captured schema, the same machinery as `// column`. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct Measure { + pub name: String, + // Aggregate SQL over the table's columns (`sum(amount)`). Trusted author text. + pub expr: String, + // Optional row predicate from a trailing `where`. Kept separate from `expr` + // so a reader can render it as an aggregate `FILTER (WHERE …)`, letting two + // measures with different filters share one GROUP BY (a shared `WHERE` + // cannot express that). + #[serde(skip_serializing_if = "Option::is_none")] + pub filter: Option, +} + +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct Dimension { + pub name: String, + // Slicing expression over the table's columns (`region`, + // `date_trunc('month', ordered_at)`). Trusted author text. + pub expr: String, +} + // `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger // firing runs the script (current behaviour). `All` = AND: the script // runs only once every partition-bearing input has materialized at the @@ -526,6 +564,8 @@ pub struct PipelineAnnotations { pub materialize: Option, pub data_tests: Vec, pub column_lineage: Vec, + pub measures: Vec, + pub dimensions: Vec, pub macros: bool, pub use_libs: Vec, // `// mute ` — suppress the auto-derived cascade edge for a read @@ -564,6 +604,8 @@ impl ParseAssetsOutput { materialize: pipeline.materialize, data_tests: pipeline.data_tests, column_lineage: pipeline.column_lineage, + measures: pipeline.measures, + dimensions: pipeline.dimensions, macros: pipeline.macros, use_libs: pipeline.use_libs, } @@ -680,26 +722,13 @@ pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(Asset } for (prefix, kind) in ASSET_KINDS.iter() { if s.starts_with(prefix) { - let path = &s[prefix.len()..]; - // Canonicalize S3 keys to a single asset identity. The SDK object - // form (`{ s3: "key" }` / `S3Object(s3="key")`, default storage) - // resolves to `s3:///key`, whose path is `/key`, while DuckDB - // `s3://key` and `// on s3://key` yield the bare `key`. Strip every - // leading slash so the triple-slash default-storage form and the - // `s3://storage/key` form share one path — otherwise a TS/Python - // writer and a DuckDB reader of the same object become disconnected - // nodes in the pipeline graph. Stripping ALL leading slashes (not - // just one) keeps the identity stable through URI reconstruction: - // `trigger_spec_to_row` rebuilds `s3://`, so a canonical path - // must never itself start with `/` or the rebuilt ref would parse - // back to a different key. Only leading slashes are touched, so - // Hive-partition keys (`s3://b/y=2024/f.parquet`) are untouched. - let path = if matches!(kind, AssetKind::S3Object) { - path.trim_start_matches('/') - } else { - path - }; - return Some((*kind, path)); + // The suffix is kept verbatim. For S3 the path encodes the storage: + // `s3:///`, with an EMPTY storage segment for the + // workspace default — so `s3:///key` yields `/key` (leading slash + // significant, default storage) while `s3://secondary/key` yields + // `secondary/key`. Stripping leading slashes here would conflate a + // default-storage object with a named-storage one. + return Some((*kind, &s[prefix.len()..])); } } None @@ -1006,6 +1035,28 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + // `// measure = [where ]` / `// dimension = + // ` — metrics primitives. Complete words (checked before the + // `on`/asset shorthands), accumulating, deduped by name (first wins). + // Malformed lines drop fail-safe like the rest of the annotation family. + if let Some(after_kw) = consume_keyword(rest, "measure") { + if let Some(spec) = parse_measure_spec(after_kw.trim()) { + if !out.measures.iter().any(|m| m.name == spec.name) { + out.measures.push(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "dimension") { + if let Some(spec) = parse_dimension_spec(after_kw.trim()) { + if !out.dimensions.iter().any(|d| d.name == spec.name) { + out.dimensions.push(spec); + } + } + continue; + } + if let Some(after_kw) = consume_keyword(rest, "on") { let spec_text = after_kw.trim(); if spec_text.is_empty() { @@ -1066,6 +1117,42 @@ pub fn count_malformed_data_tests(code: &str) -> usize { malformed } +// Count `// measure` / `// dimension` header lines whose right-hand side fails +// to parse, returning `(malformed_measures, malformed_dimensions)`. Same +// fail-safe drop as the rest of the family (a typo silently omits the metric), +// so the deploy path warns off this count. Same leading-block boundary and +// grammar as `parse_pipeline_annotations`, so "malformed" means exactly what the +// parser rejects. +pub fn count_malformed_metric_annotations(code: &str) -> (usize, usize) { + let (mut measures, mut dimensions) = (0, 0); + for raw_line in code.lines() { + let line = raw_line.trim_start(); + if line.is_empty() { + continue; + } + let rest = if let Some(r) = line.strip_prefix("//") { + r + } else if let Some(r) = line.strip_prefix("--") { + r + } else if let Some(r) = line.strip_prefix('#') { + r + } else { + break; + }; + let rest = rest.trim_start(); + if let Some(after_kw) = consume_keyword(rest, "measure") { + if parse_measure_spec(after_kw.trim()).is_none() { + measures += 1; + } + } else if let Some(after_kw) = consume_keyword(rest, "dimension") { + if parse_dimension_spec(after_kw.trim()).is_none() { + dimensions += 1; + } + } + } + (measures, dimensions) +} + // Parse a `// retry []` right-hand side. `` is a // non-negative decimal; `` is an optional raw duration string left // for `parse_duration_secs` to validate at deploy. A bare zero count (or @@ -1268,6 +1355,104 @@ fn parse_column_lineage_spec(s: &str) -> Option { Some(ColumnLineage { column, inputs }) } +// `// measure = [where ]`. The name is a single identifier; +// the first `=` separates it from the body (SQL comparison operators live after +// it, so splitting on the first `=` is unambiguous). A whitespace-bounded +// ` where ` splits the aggregate from its row filter — a `where` buried in a +// string literal or subquery would mis-split, at which point the compiled SQL +// fails at DuckDB parse (fail-loud, not silently wrong). An empty aggregate is +// rejected. Modifier metadata (format/label/additivity) is deferred: `|` and +// `||` are valid SQL operators, so a modifier delimiter must not collide with a +// measure body — that grammar is chosen when the explorer needs the fields. +fn parse_measure_spec(s: &str) -> Option { + let (name_part, body) = s.split_once('=')?; + let name = single_ident(name_part)?; + let (expr, filter) = split_measure_filter(body.trim()); + let expr = expr.trim(); + // A bare trailing `where` with no predicate is a typo, not an unfiltered + // measure: reject the whole line (counted malformed) rather than emit + // `sum(amount) where` as the expression. + if expr.is_empty() || matches!(&filter, Some(f) if f.is_empty()) { + return None; + } + Some(Measure { name, expr: expr.to_string(), filter }) +} + +// `// dimension = `. Name is a single identifier; the expression is +// trusted SQL emitted verbatim into SELECT and GROUP BY. No filter clause. +fn parse_dimension_spec(s: &str) -> Option { + let (name_part, expr) = s.split_once('=')?; + let name = single_ident(name_part)?; + let expr = expr.trim(); + if expr.is_empty() { + return None; + } + Some(Dimension { name, expr: expr.to_string() }) +} + +// Split a measure body on the first top-level whitespace-bounded ` where` into +// (aggregate, filter). "Top-level" = outside string/quoted-identifier literals and +// outside parentheses, so `count_if(note = 'some where value')` is not split on the +// `where` buried in its string. `to_ascii_lowercase` is byte-length-preserving, so +// an offset in the lowercased copy indexes the original; the predicate keeps its +// source casing. A trailing ` where` yields an empty predicate, which the caller +// rejects, so `sum(amount) where` is not read as a filterless aggregate. +fn split_measure_filter(body: &str) -> (&str, Option) { + // Byte scan, comparing on the ASCII-lowercased bytes: `to_ascii_lowercase` is + // byte-length-preserving, and byte-slice matching never panics on a non-UTF-8 + // boundary the way `lower[i..]` would for a non-ASCII body (`sum(x) + π`). + let lower = body.to_ascii_lowercase(); + let lb = lower.as_bytes(); + let b = body.as_bytes(); + let (mut in_single, mut in_double) = (false, false); + let mut depth: i32 = 0; + let mut i = 0usize; + while i < b.len() { + let c = b[i]; + if in_single { + if c == b'\'' { + // A doubled quote is an escaped one, not the end. + if b.get(i + 1) == Some(&b'\'') { + i += 2; + continue; + } + in_single = false; + } + i += 1; + continue; + } + if in_double { + if c == b'"' { + if b.get(i + 1) == Some(&b'"') { + i += 2; + continue; + } + in_double = false; + } + i += 1; + continue; + } + match c { + b'\'' => in_single = true, + b'"' => in_double = true, + b'(' => depth += 1, + b')' => depth -= 1, + _ if depth == 0 && lb[i..].starts_with(b" where") => { + let after = i + b" where".len(); + // Whitespace-bounded: end of input, or a space follows (so a token + // like ` wherever` is not matched). `i` is the space before `where` + // and `after` is just past it — both ASCII, so the slices are valid. + if after == b.len() || b[after].is_ascii_whitespace() { + return (&body[..i], Some(body[after..].trim().to_string())); + } + } + _ => {} + } + i += 1; + } + (body, None) +} + // `.` — the referenced column is the segment after the final // `.`; everything before it is the asset URI (default-syntax shorthands // enabled, like `// materialize`). Same shape as `parse_relationships`' target. @@ -1330,6 +1515,7 @@ fn parse_trigger_spec(s: &str) -> Option { ("email", TriggerSpec::Email), ("kafka", TriggerSpec::Kafka), ("mqtt", TriggerSpec::Mqtt), + ("amqp", TriggerSpec::Amqp), ("nats", TriggerSpec::Nats), ("postgres", TriggerSpec::Postgres), ("sqs", TriggerSpec::Sqs), @@ -1361,52 +1547,43 @@ mod pipeline_annotation_tests { use super::*; #[test] - fn s3_key_normalization_unifies_uri_forms() { - // A TS/Python SDK write of `{ s3: "exports/x" }` (default storage) - // resolves to the URI `s3:///exports/x`, while a DuckDB read of - // `s3://exports/x` and the `// on s3://exports/x` trigger form yield the - // bare `exports/x`. All three must canonicalize to one asset key so - // the writer and reader connect in the pipeline graph. - let sdk_write = parse_asset_syntax("s3:///exports/x", false); - let duckdb_read = parse_asset_syntax("s3://exports/x", false); - assert_eq!(sdk_write, Some((AssetKind::S3Object, "exports/x"))); - assert_eq!(duckdb_read, Some((AssetKind::S3Object, "exports/x"))); - assert_eq!(sdk_write, duckdb_read); + fn s3_path_keeps_storage_distinction() { + // An S3 asset path is `/` with an empty storage segment + // for the workspace default. The default-storage form `s3:///key` + // yields `/key` (leading slash significant); the named-storage form + // `s3://secondary/key` yields `secondary/key`. The two name DIFFERENT + // objects and must never collapse to one identity. + assert_eq!( + parse_asset_syntax("s3:///exports/x", false), + Some((AssetKind::S3Object, "/exports/x")) + ); + assert_eq!( + parse_asset_syntax("s3://exports/x", false), + Some((AssetKind::S3Object, "exports/x")) + ); + assert_ne!( + parse_asset_syntax("s3:///exports/x", false), + parse_asset_syntax("s3://exports/x", false) + ); // The `// on` trigger annotation goes through the same function. assert_eq!( parse_asset_syntax("s3:///exports/x", true), - parse_asset_syntax("s3://exports/x", true) + Some((AssetKind::S3Object, "/exports/x")) ); - // Explicit-storage form is unaffected (no leading slash to strip). assert_eq!( - parse_asset_syntax("s3://mybucket/exports/x", false), - Some((AssetKind::S3Object, "mybucket/exports/x")) + parse_asset_syntax("s3://secondary_storage/path/to/file.csv", false), + Some((AssetKind::S3Object, "secondary_storage/path/to/file.csv")) ); - // Hive-partition keys and nested paths under default storage are - // preserved verbatim (only leading slashes are stripped). + // Hive-partition keys are preserved verbatim. assert_eq!( parse_asset_syntax("s3:///t/year=2024/month=01/f.parquet", false), - Some((AssetKind::S3Object, "t/year=2024/month=01/f.parquet")) + Some((AssetKind::S3Object, "/t/year=2024/month=01/f.parquet")) ); - // Every leading slash is stripped so a canonical S3 path never starts - // with `/`. `S3Object(s3="/x")` resolves to the quad-slash URI - // `s3:////x`; the identity must be the bare `x` (not `/x`) so the ref - // that `trigger_spec_to_row` rebuilds round-trips back to it. - assert_eq!( - parse_asset_syntax("s3:////x", false), - Some((AssetKind::S3Object, "x")) - ); - assert_eq!( - parse_asset_syntax("s3://///deep///", false), - Some((AssetKind::S3Object, "deep///")) - ); - - // Non-S3 kinds keep their leading slash (their paths are workspace- - // relative and the slash is significant). + // Non-S3 kinds also keep their suffix verbatim. assert_eq!( parse_asset_syntax("res://f/foo", false), Some((AssetKind::Resource, "f/foo")) @@ -1417,26 +1594,6 @@ mod pipeline_annotation_tests { ); } - #[test] - fn s3_explicit_storage_aliases_default_storage_nested_key() { - // Accepted tradeoff of one canonical key: the explicit-storage form - // `s3://storage/key` and the default-storage nested-key form - // `s3:///storage/key` collapse to the same node `storage/key`, even - // though they name different objects. This is a best-effort lineage - // graph that does not split the first segment as a storage name; the - // collision only happens when a storage config is named to match a - // default-storage prefix. Pinned so the aliasing is intentional, not a - // latent surprise. - assert_eq!( - parse_asset_syntax("s3://mybucket/x", false), - parse_asset_syntax("s3:///mybucket/x", false) - ); - assert_eq!( - parse_asset_syntax("s3://mybucket/x", false), - Some((AssetKind::S3Object, "mybucket/x")) - ); - } - #[test] fn bare_pipeline_marker() { let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')"); @@ -1472,6 +1629,99 @@ mod pipeline_annotation_tests { assert!(!parse_pipeline_annotations("// macros_v2\n").macros); } + #[test] + fn measures_and_dimensions_parse() { + let out = parse_pipeline_annotations( + "-- materialize ducklake://f/finance/orders\n\ + -- measure revenue = sum(amount) where not is_refund\n\ + -- measure orders = count(*)\n\ + -- dimension region = region\n\ + -- dimension month = date_trunc('month', ordered_at)\n\ + SELECT 1;", + ); + assert_eq!( + out.measures, + vec![ + Measure { + name: "revenue".to_string(), + expr: "sum(amount)".to_string(), + filter: Some("not is_refund".to_string()), + }, + Measure { name: "orders".to_string(), expr: "count(*)".to_string(), filter: None }, + ] + ); + assert_eq!( + out.dimensions, + vec![ + Dimension { name: "region".to_string(), expr: "region".to_string() }, + Dimension { + name: "month".to_string(), + expr: "date_trunc('month', ordered_at)".to_string(), + }, + ] + ); + } + + #[test] + fn a_where_inside_a_string_or_parens_is_not_a_filter_delimiter() { + // The `where` is inside the aggregate's string argument, so the whole + // expression is the measure and there is no filter. + assert_eq!( + split_measure_filter("count_if(note = 'some where value')"), + ("count_if(note = 'some where value')", None) + ); + // A real top-level filter still splits. + assert_eq!( + split_measure_filter("sum(amount) where not is_refund"), + ("sum(amount)", Some("not is_refund".to_string())) + ); + // A `where` inside parens is not the delimiter either. + assert_eq!( + split_measure_filter("sum(case when x then 1 end)"), + ("sum(case when x then 1 end)", None) + ); + // A non-ASCII body must not panic on a byte offset that is not a char + // boundary. + assert_eq!( + split_measure_filter("sum(amount) + π"), + ("sum(amount) + π", None) + ); + assert_eq!( + split_measure_filter("sum(π) where region = 'π'"), + ("sum(π)", Some("region = 'π'".to_string())) + ); + } + + #[test] + fn measures_dimensions_fail_safe_and_dedup() { + let out = parse_pipeline_annotations( + // Malformed (no `=`, empty body, bare trailing `where`) drop; a + // duplicate name keeps the first; the body comment after real code + // is never scanned. + "-- measure broken\n\ + -- measure empty = \n\ + -- measure dangling = sum(amount) where\n\ + -- measure revenue = sum(amount)\n\ + -- measure revenue = sum(other)\n\ + -- dimension region = region\n\ + SELECT 1;\n\ + -- measure sneaky = count(*)\n\ + -- dimension sneaky = x", + ); + assert_eq!( + out.measures, + vec![Measure { + name: "revenue".to_string(), + expr: "sum(amount)".to_string(), + filter: None, + }] + ); + assert_eq!( + out.dimensions, + vec![Dimension { name: "region".to_string(), expr: "region".to_string() }] + ); + } + #[test] fn use_accumulates_dedups_and_rejects_prose() { let out = parse_pipeline_annotations( diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 226f32914c..a09349647d 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -741,12 +741,12 @@ } }, { - "name": "s3 triple-slash default-storage trigger canonicalizes to bare key", + "name": "s3 triple-slash default-storage trigger keeps its leading slash", "code": "// pipeline\n// on s3:///exports/x\nexport function main() {}", "expected": { "in_pipeline": true, "asset_triggers": [ - "s3object:exports/x" + "s3object:/exports/x" ], "native_triggers": [], "partition": null, @@ -756,12 +756,12 @@ } }, { - "name": "s3 quad-slash trigger strips all leading slashes to the bare key", - "code": "// pipeline\n// on s3:////x\nexport function main() {}", + "name": "s3 named-storage trigger keeps the storage segment", + "code": "// pipeline\n// on s3://secondary_storage/exports/x\nexport function main() {}", "expected": { "in_pipeline": true, "asset_triggers": [ - "s3object:x" + "s3object:secondary_storage/exports/x" ], "native_triggers": [], "partition": null, diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index fff818edb2..1936f7ff70 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -121,6 +121,7 @@ fn native_str(t: &TriggerSpec) -> Option<&'static str> { TriggerSpec::Email => "email", TriggerSpec::Kafka => "kafka", TriggerSpec::Mqtt => "mqtt", + TriggerSpec::Amqp => "amqp", TriggerSpec::Nats => "nats", TriggerSpec::Postgres => "postgres", TriggerSpec::Sqs => "sqs", diff --git a/backend/src/main.rs b/backend/src/main.rs index 43502a1980..aa895df3e7 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -40,14 +40,14 @@ use windmill_common::{ global_settings::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, BUN_INSTALL_MIN_RELEASE_AGE_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, - INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + CONCURRENCY_KEY_MAX_QUEUED_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, + INDEXER_SETTING, INSTANCE_EVENTS_WEBHOOK_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -64,7 +64,7 @@ use windmill_common::{ UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_REGISTRIES_SETTING, + WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, }, scripts::ScriptLang, stats_oss::schedule_stats, @@ -122,12 +122,13 @@ use windmill_worker::{ }; use crate::monitor::{ - initial_load, load_disable_password_login, load_fork_workspace_tag_append_fork_suffix, - load_keep_job_dir, load_metrics_debug_enabled, load_preview_tags_override, - load_require_preexisting_user, load_retention_period_overrides, load_tag_per_workspace_enabled, - load_tag_per_workspace_workspaces, load_workspace_fairness_duration_secs, - load_workspace_fairness_enabled, load_workspace_fairness_max_percent, - load_workspace_fairness_min_total, monitor_db, reload_app_workspaced_route_setting, + initial_load, load_concurrency_key_max_queued, load_disable_password_login, + load_fork_workspace_tag_append_fork_suffix, load_keep_job_dir, load_metrics_debug_enabled, + load_preview_tags_override, load_require_preexisting_user, load_retention_period_overrides, + load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, + load_workspace_fairness_duration_secs, load_workspace_fairness_enabled, + load_workspace_fairness_max_percent, load_workspace_fairness_min_total, + load_workspace_max_queued_jobs, monitor_db, reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting, reload_base_url_setting, reload_bun_install_min_release_age_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting, @@ -289,11 +290,18 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { create_dir_all(&*HUB_CACHE_DIR)?; create_dir_all(&*BUN_BUNDLE_CACHE_DIR)?; - // Ensure the latest git sync script is always cached, regardless of hubPaths.json contents + // Ensure the backend-hardcoded git sync scripts are always cached, regardless of + // hubPaths.json contents. These are run by backend-driven sync (not necessarily listed + // in hubPaths.json), so airgapped workers would otherwise miss them on a cache lookup. let mut all_paths: Vec = paths.into_values().collect(); - let latest_git_sync = windmill_common::workspaces::LATEST_GIT_SYNC_SCRIPT_PATH.to_string(); - if !all_paths.contains(&latest_git_sync) { - all_paths.push(latest_git_sync); + for git_sync_path in [ + windmill_common::workspaces::LATEST_GIT_SYNC_SCRIPT_PATH, + windmill_common::workspaces::GIT_SYNC_PULL_SCRIPT_PATH, + ] { + let git_sync_path = git_sync_path.to_string(); + if !all_paths.contains(&git_sync_path) { + all_paths.push(git_sync_path); + } } for path in &all_paths { @@ -1879,6 +1887,16 @@ async fn process_notify_event( tracing::error!("Error loading workspace fairness min total: {e:#}"); } } + CONCURRENCY_KEY_MAX_QUEUED_SETTING => { + if let Err(e) = load_concurrency_key_max_queued(db).await { + tracing::error!("Error loading concurrency key max queued: {e:#}"); + } + } + WORKSPACE_MAX_QUEUED_JOBS_SETTING => { + if let Err(e) = load_workspace_max_queued_jobs(db).await { + tracing::error!("Error loading workspace max queued jobs: {e:#}"); + } + } SMTP_SETTING => { reload_smtp_config(db).await; } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6632e3b3a8..adf859d139 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -53,11 +53,12 @@ use windmill_common::{ flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, + BUN_INSTALL_MIN_RELEASE_AGE_SETTING, CONCURRENCY_KEY_MAX_QUEUED_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, EXPOSE_DEBUG_METRICS_SETTING, + EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, @@ -73,6 +74,7 @@ use windmill_common::{ UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, @@ -86,11 +88,13 @@ use windmill_common::{ load_env_vars, load_init_bash_from_env, load_periodic_bash_script_from_env, load_periodic_bash_script_interval_from_env, load_whitelist_env_vars_from_env, load_worker_config, reload_custom_tags_setting, store_pull_query, - store_suspended_pull_query, Connection, WorkerConfig, DEFAULT_TAGS_PER_WORKSPACE, + store_suspended_pull_query, Connection, WorkerConfig, CLOUD_HOSTED, + CONCURRENCY_KEY_MAX_QUEUED, CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX, INDEXER_CONFIG, PREVIEW_TAGS_OVERRIDE, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, WORKSPACE_FAIRNESS_DURATION_SECS, WORKSPACE_FAIRNESS_ENABLED, - WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, + WORKSPACE_FAIRNESS_MAX_PERCENT, WORKSPACE_FAIRNESS_MIN_TOTAL, WORKSPACE_MAX_QUEUED_JOBS, + WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, }, KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, @@ -108,7 +112,11 @@ use windmill_common::{ }; #[cfg(feature = "parquet")] use windmill_object_store::reload_object_store_setting; -use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; +use windmill_queue::{ + cancel_job, get_queued_job_v2, + schedule::{find_unarmed_schedules, rearm_schedule, RearmOutcome}, + SameWorkerPayload, +}; use windmill_worker::{ result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel, OtelTracingProxySettings, SameWorkerSender, WorkspaceRegistryMap, BUNFIG_INSTALL_SCOPES, @@ -287,6 +295,16 @@ pub async fn initial_load( if let Err(e) = load_workspace_fairness_enabled(db).await { tracing::error!("Error loading workspace fairness enabled: {e:#}"); } + + // Only the cloud reads this cap, so don't spend a query loading it anywhere else. + if *CLOUD_HOSTED { + if let Err(e) = load_concurrency_key_max_queued(db).await { + tracing::error!("Error loading concurrency key max queued: {e:#}"); + } + if let Err(e) = load_workspace_max_queued_jobs(db).await { + tracing::error!("Error loading workspace max queued jobs: {e:#}"); + } + } } if server_mode { @@ -606,6 +624,10 @@ const WORKSPACE_FAIRNESS_MAX_PERCENT_DEFAULT: u32 = 50; const WORKSPACE_FAIRNESS_DURATION_SECS_DEFAULT: u32 = 10; const WORKSPACE_FAIRNESS_MIN_TOTAL_DEFAULT: u32 = 4; +/// The cap is used as a SQL `LIMIT`, so it must survive the `u32 -> i64` widening without +/// becoming absurd; `u32::MAX` is already far beyond any queue depth worth allowing. +const CONCURRENCY_KEY_MAX_QUEUED_MAX: u64 = u32::MAX as u64; + pub async fn load_workspace_fairness_enabled(db: &DB) -> error::Result<()> { // Match the convention used by `load_preview_tags_override` / // `load_fork_workspace_tag_append_fork_suffix`: on transient DB errors, leave the in-memory @@ -693,6 +715,72 @@ pub async fn load_workspace_fairness_min_total(db: &DB) -> error::Result<()> { Ok(()) } +pub async fn load_concurrency_key_max_queued(db: &DB) -> error::Result<()> { + // See `load_workspace_fairness_max_percent` for the Err / None / invalid policy. + match load_value_from_global_settings(db, CONCURRENCY_KEY_MAX_QUEUED_SETTING).await? { + Some(serde_json::Value::Number(n)) => { + // `0` is a meaningful value here (disable the cap), so unlike the fairness knobs + // the lower bound is 0 rather than 1. + let v = n + .as_u64() + .map(|u| u.min(CONCURRENCY_KEY_MAX_QUEUED_MAX) as u32) + .unwrap_or_else(|| { + // Warn rather than silently defaulting: `-1` and `"0"` are plausible + // attempts to disable the cap, and both would otherwise land on 10000. + tracing::warn!( + "{CONCURRENCY_KEY_MAX_QUEUED_SETTING}={n} is not a non-negative integer, \ + falling back to {CONCURRENCY_KEY_MAX_QUEUED_DEFAULT}. Set 0 to disable." + ); + CONCURRENCY_KEY_MAX_QUEUED_DEFAULT + }); + CONCURRENCY_KEY_MAX_QUEUED.store(v, Ordering::Relaxed); + } + other => { + if let Some(v) = other { + tracing::warn!( + "{CONCURRENCY_KEY_MAX_QUEUED_SETTING}={v} is not a number, falling back to \ + {CONCURRENCY_KEY_MAX_QUEUED_DEFAULT}. Set 0 to disable." + ); + } + CONCURRENCY_KEY_MAX_QUEUED.store(CONCURRENCY_KEY_MAX_QUEUED_DEFAULT, Ordering::Relaxed); + } + } + Ok(()) +} + +pub async fn load_workspace_max_queued_jobs(db: &DB) -> error::Result<()> { + // Only the cloud enforces this cap, so never spend the query off-cloud, from any call site. + if !*CLOUD_HOSTED { + return Ok(()); + } + // Same Err / None / invalid policy as load_concurrency_key_max_queued: 0 disables. + match load_value_from_global_settings(db, WORKSPACE_MAX_QUEUED_JOBS_SETTING).await? { + Some(serde_json::Value::Number(n)) => { + let v = n + .as_u64() + .map(|u| u.min(CONCURRENCY_KEY_MAX_QUEUED_MAX) as u32) + .unwrap_or_else(|| { + tracing::warn!( + "{WORKSPACE_MAX_QUEUED_JOBS_SETTING}={n} is not a non-negative integer, \ + falling back to {WORKSPACE_MAX_QUEUED_JOBS_DEFAULT}. Set 0 to disable." + ); + WORKSPACE_MAX_QUEUED_JOBS_DEFAULT + }); + WORKSPACE_MAX_QUEUED_JOBS.store(v, Ordering::Relaxed); + } + other => { + if let Some(v) = other { + tracing::warn!( + "{WORKSPACE_MAX_QUEUED_JOBS_SETTING}={v} is not a number, falling back to \ + {WORKSPACE_MAX_QUEUED_JOBS_DEFAULT}. Set 0 to disable." + ); + } + WORKSPACE_MAX_QUEUED_JOBS.store(WORKSPACE_MAX_QUEUED_JOBS_DEFAULT, Ordering::Relaxed); + } + } + Ok(()) +} + pub async fn load_fork_workspace_tag_append_fork_suffix(db: &DB) -> error::Result<()> { let value = load_value_from_global_settings(db, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING).await; @@ -1061,12 +1149,16 @@ pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) { if current.enabled != new_settings.enabled || current.enabled_languages != new_settings.enabled_languages || current.no_proxy_hosts != new_settings.no_proxy_hosts + || current.insecure_upstream_hosts != new_settings.insecure_upstream_hosts + || current.upstream_ca_certs != new_settings.upstream_ca_certs { tracing::info!( - "OTEL tracing proxy settings changed: enabled={}, languages={:?}, no_proxy_hosts={:?}", + "OTEL tracing proxy settings changed: enabled={}, languages={:?}, no_proxy_hosts={:?}, insecure_upstream_hosts={:?}, upstream_ca_certs={}", new_settings.enabled, new_settings.enabled_languages, new_settings.no_proxy_hosts, + new_settings.insecure_upstream_hosts, + if new_settings.upstream_ca_certs.as_deref().unwrap_or("").trim().is_empty() { "unset" } else { "set" }, ); *current = new_settings; } @@ -1308,6 +1400,16 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error reaping stale join_pending_inputs slots: {:?}", e); } + // 60-day retention for anonymous feature-usage counters. Runs here (not only + // in the telemetry sender) so rows are pruned even when telemetry is disabled + // or the build has no stats scheduler. + if let Err(e) = sqlx::query!("DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60") + .execute(db) + .await + { + tracing::error!("Error deleting old feature_usage rows: {e}"); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) @@ -3091,6 +3193,14 @@ pub async fn monitor_db( if !initial_load { verify_license_key(conn.as_sql()).await; refetch_license_key_if_invalid(conn).await; + // Server-side only: the alert writes to the alerts table and notifies + // the critical channels, so gate it like enforce_offline_caps rather + // than have every worker re-report the same expiry. + if server_mode { + if let Some(db) = conn.as_sql() { + windmill_common::ee_oss::alert_on_online_license_expired(db).await; + } + } } }; @@ -3255,9 +3365,10 @@ pub async fn monitor_db( } }; - // run every ~60s (2 iterations * 30s). Enterprise feature: core logic is - // in `crate::ee` (OSS gets a no-op stub); gated on a valid Enterprise - // license, mirroring how `audit_log()` itself is license-aware. + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). + // Enterprise feature: core logic is in `crate::ee` (OSS gets a no-op stub); + // gated on a valid Enterprise license, mirroring how `audit_log()` itself + // is license-aware. let export_audit_logs_to_object_store_f = async { #[cfg(feature = "parquet")] if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(2) { @@ -3281,11 +3392,32 @@ pub async fn monitor_db( } }; - // run every ~60s (2 iterations * 30s). Enterprise feature: the active - // `// freshness` backstop lives in windmill-queue's `freshness_watchdog` - // (`private`); OSS gets a no-op stub. Runtime-gated on an Enterprise - // license like the audit export above. Safe on concurrent servers — the - // watchdog claims per-script state rows atomically before pushing. + // run every 30 iterations (~5min at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). + let reconcile_unarmed_schedules_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(30) { + if let Some(db) = conn.as_sql() { + reconcile_unarmed_schedules(&db).await; + } + } + }; + + // Poll git-sync repositories for new commits and pull them into the + // workspace (repo → Windmill auto-pull). Runs every 2 iterations. + let git_auto_pull_f = async { + #[cfg(feature = "private")] + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(2) { + if let Some(db) = conn.as_sql() { + poll_git_auto_pull(db).await; + } + } + }; + + // run every 2 iterations (~20s at the default LISTEN_NEW_EVENTS_INTERVAL_SEC). + // Enterprise feature: the active `// freshness` backstop lives in + // windmill-queue's `freshness_watchdog` (`private`); OSS gets a no-op stub. + // Runtime-gated on an Enterprise license like the audit export above. Safe + // on concurrent servers — the watchdog claims per-script state rows + // atomically before pushing. let pipeline_freshness_watchdog_f = async { if server_mode && !*DISABLE_FRESHNESS_WATCHDOG @@ -3330,10 +3462,567 @@ pub async fn monitor_db( manage_audit_partitions_f, export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, + git_auto_pull_f, pipeline_freshness_watchdog_f, + reconcile_unarmed_schedules_f, ); } +/// Advisory lock id ensuring only one server replica reconciles schedules at a +/// time (adjacent to GIT_AUTO_PULL_LOCK_ID). +const SCHEDULE_RECONCILE_LOCK_ID: i64 = 737_483_922; + +/// Consecutive reconciliation passes an enabled schedule must be observed with no +/// queued occurrence before it is re-armed. The next occurrence is pushed in the +/// same transaction that completes the previous one (or, for flows, on entry to +/// step 0), so an unarmed schedule is normally only ever a mid-flight push or a +/// push being retried. Requiring two passes keeps the reconciler from racing +/// those and double-pushing an occurrence. +const SCHEDULE_RECONCILE_STRIKES: u8 = 2; + +/// Most schedules re-armed in one pass, so a large first-pass backlog is drained +/// over several passes instead of enqueuing every occurrence at once. +const SCHEDULE_RECONCILE_MAX_PER_PASS: usize = 50; + +/// Consecutive failed re-arm attempts after which a schedule's persistent failure +/// is surfaced once (its `error` recorded + a critical alert). Most re-arm +/// failures are a transient blip that clears on the next attempt; a schedule that +/// keeps failing has a real cause (bad stored cron/timezone/args, lapsed license) +/// and would otherwise be enabled-yet-silently-dead. +const SCHEDULE_REARM_ALERT_THRESHOLD: u32 = 3; + +/// Cap on the exponential back-off (in reconcile passes) between re-arm retries of +/// a schedule that keeps failing. Without a back-off a permanently-failing push +/// retries every pass forever; the delay grows 2, 4, 8 and holds at this cap, and +/// resets the moment the schedule re-arms. Kept small so a schedule fixed out of +/// band (the UI/API re-arms immediately) still auto-recovers within a few passes. +const SCHEDULE_REARM_MAX_BACKOFF_PASSES: u32 = 8; + +/// State for a schedule that keeps failing to re-arm: paces retries and drives the +/// one-time visibility so the loop is neither hot nor silent. +#[derive(Default)] +struct RearmFailureState { + consecutive_failures: u32, + /// Reconcile passes still to skip before the next attempt (exponential back-off). + cooldown_passes: u32, + /// Whether the persistent failure has already been surfaced. + surfaced: bool, +} + +lazy_static::lazy_static! { + /// `(workspace_id, path)` -> consecutive passes seen with no queued occurrence. + /// Bounded by the number of enabled schedules; entries drop as soon as a + /// schedule is seen armed again. + static ref UNARMED_SCHEDULES: Mutex> = + Mutex::new(std::collections::HashMap::new()); + + /// `(workspace_id, path)` -> back-off/visibility state for schedules that keep + /// failing to re-arm. Entries drop as soon as a schedule re-arms or is no + /// longer unarmed (armed, disabled, deleted). + static ref SCHEDULE_REARM_FAILURES: Mutex> = + Mutex::new(std::collections::HashMap::new()); +} + +/// Re-arm enabled schedules that have no queued occurrence. +/// +/// Every path that completes a scheduled job is supposed to push the next +/// occurrence atomically, but a run that dies through an abnormal path (a flow +/// whose status update fails and is later force-completed by zombie detection, +/// say) can skip that push and leave the schedule enabled yet dead forever. This +/// is the backstop: without it the only recovery is a manual disable/enable. +/// +/// Replicas each run their own passes (staggered by `rd_shift`) and each keep +/// their own strike tally, so the scan cost is per-replica. That is deliberate: +/// scanning inside the advisory lock is what makes a double-push impossible — +/// whoever holds it re-reads the unarmed set, so a schedule another replica just +/// re-armed is seen armed and its tally dropped, rather than pushed twice. +/// +/// Not an authorization boundary: it re-arms schedules across every workspace, so +/// this is a system caller (the monitor loop) only. +async fn reconcile_unarmed_schedules(db: &Pool) { + // Transaction-scoped advisory lock, not session-scoped: monitor_db runs under a + // 600s timeout, and if it fires the whole future is dropped mid-pass. A session + // lock taken on a pooled connection would then ride that connection back into the + // pool still held, wedging reconciliation on every replica until the process + // restarts. An xact lock is released when its transaction ends — including the + // rollback a dropped `Transaction` performs — so cancellation can't strand it. + // The tx is held open only to own the lock; the scan and re-arm run on separate + // pool connections. + let mut lock_tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("schedule reconcile: failed to begin lock tx: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(SCHEDULE_RECONCILE_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("schedule reconcile: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + // Another replica is already reconciling this tick. + return; + } + + if let Err(e) = reconcile_unarmed_schedules_inner(db).await { + tracing::error!("schedule reconcile: {e:#}"); + } + + // Ends the transaction and releases the xact lock; a plain drop would too. + if let Err(e) = lock_tx.rollback().await { + tracing::error!("schedule reconcile: releasing lock failed: {e:#}"); + } +} + +/// Record this pass's unarmed schedules against `seen` and return those that have +/// now struck out. An armed observation drops the schedule's tally entirely, so +/// the strikes a re-arm rests on are always consecutive. +fn strike_unarmed( + seen: &mut std::collections::HashMap<(String, String), u8>, + current: std::collections::HashSet<(String, String)>, +) -> Vec<(String, String)> { + seen.retain(|k, _| current.contains(k)); + current + .into_iter() + .filter(|k| { + let strikes = seen.entry(k.clone()).or_insert(0); + *strikes = strikes.saturating_add(1); + *strikes >= SCHEDULE_RECONCILE_STRIKES + }) + .collect() +} + +async fn reconcile_unarmed_schedules_inner(db: &Pool) -> error::Result<()> { + let current: std::collections::HashSet<(String, String)> = + find_unarmed_schedules(db).await?.into_iter().collect(); + let mut to_rearm = strike_unarmed(&mut UNARMED_SCHEDULES.lock().unwrap(), current.clone()); + + // Back off schedules that keep failing to re-arm (bad stored cron/timezone/args, + // lapsed license): skip those still cooling down, and forget state for any that + // are no longer unarmed. Without this a permanently-failing push is retried + // every pass forever. + { + let mut failures = SCHEDULE_REARM_FAILURES.lock().unwrap(); + failures.retain(|k, _| current.contains(k)); + to_rearm.retain(|k| match failures.get_mut(k) { + Some(state) if state.cooldown_passes > 0 => { + state.cooldown_passes -= 1; + false + } + _ => true, + }); + } + + // The first pass on an instance that has never been swept can find a large + // backlog; re-arming it all at once would enqueue that whole backlog in one + // go. The overflow keeps its tally and is picked up next pass. + if to_rearm.len() > SCHEDULE_RECONCILE_MAX_PER_PASS { + tracing::warn!( + "schedule reconcile: {} schedules have no queued occurrence, re-arming {} this pass and the rest on later passes", + to_rearm.len(), + SCHEDULE_RECONCILE_MAX_PER_PASS + ); + to_rearm.truncate(SCHEDULE_RECONCILE_MAX_PER_PASS); + } + + for (w_id, path) in to_rearm { + match rearm_schedule(db, &w_id, &path).await { + Ok(outcome) => { + if outcome == RearmOutcome::Rearmed { + tracing::warn!( + "schedule reconcile: re-armed enabled schedule {path} in {w_id}, which had no queued occurrence" + ); + } + UNARMED_SCHEDULES + .lock() + .unwrap() + .remove(&(w_id.clone(), path.clone())); + // Clear the error we recorded once it re-arms, so a recovered + // schedule stops showing a stale failure. + let was_surfaced = SCHEDULE_REARM_FAILURES + .lock() + .unwrap() + .remove(&(w_id.clone(), path.clone())) + .is_some_and(|s| s.surfaced); + if was_surfaced { + if let Err(e) = sqlx::query!( + "UPDATE schedule SET error = NULL WHERE workspace_id = $1 AND path = $2 AND enabled IS TRUE", + w_id, + path + ) + .execute(db) + .await + { + tracing::error!( + "schedule reconcile: could not clear error for {path} in {w_id}: {e:#}" + ); + } + } + } + Err(e) => { + tracing::error!( + "schedule reconcile: could not re-arm schedule {path} in {w_id}: {e:#}" + ); + let should_surface = { + let mut failures = SCHEDULE_REARM_FAILURES.lock().unwrap(); + let state = failures.entry((w_id.clone(), path.clone())).or_default(); + state.consecutive_failures += 1; + state.cooldown_passes = (1u32 << state.consecutive_failures.min(5)) + .min(SCHEDULE_REARM_MAX_BACKOFF_PASSES); + let surface = !state.surfaced + && state.consecutive_failures >= SCHEDULE_REARM_ALERT_THRESHOLD; + state.surfaced |= surface; + surface + }; + // Surface without disabling: record the error for the owner (schedule + // stays enabled) and raise a critical alert. rearm_schedule never + // disables, so this is the only signal a persistently-broken schedule + // gives beyond server logs. + if should_surface { + if let Err(err) = sqlx::query!( + "UPDATE schedule SET error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled IS TRUE", + e.to_string(), + w_id, + path + ) + .execute(db) + .await + { + tracing::error!( + "schedule reconcile: could not record error for {path} in {w_id}: {err:#}" + ); + } + report_critical_error( + format!( + "Schedule {path} in workspace {w_id} is enabled but has repeatedly failed to re-arm and will not run until the cause is fixed: {e:#}" + ), + db.clone(), + Some(&w_id), + None, + ) + .await; + } + } + } + } + Ok(()) +} + +/// Advisory lock id ensuring only one server replica runs the git auto-pull +/// poll at a time (adjacent to RESTART_LOCK_ID used for restart coordination). +#[cfg(feature = "private")] +const GIT_AUTO_PULL_LOCK_ID: i64 = 737_483_921; + +/// Poll every git-sync repository with auto-pull enabled and enqueue a pull when +/// the tracked branch has new commits (repo → Windmill direction). +/// +/// Runs on a single replica at a time (advisory lock) and only on +/// Enterprise-licensed instances. Detection is `git ls-remote`; GitHub-App +/// repositories are skipped here and sync via webhooks instead (phase 2). +#[cfg(feature = "private")] +pub async fn poll_git_auto_pull(db: &Pool) { + use windmill_common::ee_oss::{get_license_plan, LicensePlan}; + + if !matches!(get_license_plan().await, LicensePlan::Enterprise) { + return; + } + + let mut lock_conn = match db.acquire().await { + Ok(c) => c, + Err(e) => { + tracing::error!("git auto-pull: failed to acquire connection: {e:#}"); + return; + } + }; + let locked: bool = match sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(GIT_AUTO_PULL_LOCK_ID) + .fetch_one(&mut *lock_conn) + .await + { + Ok(v) => v, + Err(e) => { + tracing::error!("git auto-pull: advisory lock failed: {e:#}"); + return; + } + }; + if !locked { + // Another replica is already polling this tick. + return; + } + + if let Err(e) = poll_git_auto_pull_inner(db).await { + tracing::error!("git auto-pull: poll error: {e:#}"); + } + + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(GIT_AUTO_PULL_LOCK_ID) + .execute(&mut *lock_conn) + .await + { + tracing::error!("git auto-pull: advisory unlock failed: {e:#}"); + } +} + +#[cfg(feature = "private")] +lazy_static::lazy_static! { + /// Last auto-pull poll time (unix secs) per `workspace|repo_path`, so each repo + /// is only probed once per its effective interval instead of every ~60s tick. + /// Bounded by the number of auto-pull repos; stale entries for removed repos are + /// harmless. + static ref AUTO_PULL_LAST_POLL: std::sync::Mutex> = + std::sync::Mutex::new(std::collections::HashMap::new()); +} + +/// Slack (seconds) subtracted from the effective interval so a repo whose interval +/// equals the ~60s tick isn't skipped by tick jitter. +#[cfg(feature = "private")] +const AUTO_PULL_POLL_SLACK_S: i64 = 30; + +#[cfg(feature = "private")] +async fn poll_git_auto_pull_inner(db: &Pool) -> error::Result<()> { + use windmill_common::workspaces::{AutoPullMode, WorkspaceGitSyncSettings}; + + // Join `workspace` and skip deleted/archived ones: their `workspace_settings` + // rows persist (archive is a soft delete, and change_workspace_id leaves the old + // id as an archived shell), so an auto-pull repo would otherwise keep polling and + // deploying into a dead workspace. + let rows = sqlx::query!( + r#"SELECT ws.workspace_id, ws.git_sync + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE NOT w.deleted + AND ws.git_sync IS NOT NULL + AND ws.git_sync->'repositories' @> '[{"auto_pull": {"enabled": true}}]'::jsonb"# + ) + .fetch_all(db) + .await?; + + for row in rows { + let Some(git_sync) = row.git_sync else { + continue; + }; + let settings: WorkspaceGitSyncSettings = match serde_json::from_value(git_sync) { + Ok(s) => s, + Err(e) => { + tracing::warn!( + "git auto-pull: invalid git_sync settings for workspace {}: {e}", + row.workspace_id + ); + continue; + } + }; + + for repo in &settings.repositories { + let Some(auto_pull) = &repo.auto_pull else { + continue; + }; + if !auto_pull.enabled || auto_pull.mode == AutoPullMode::Webhook { + continue; + } + + // Honor the repo's effective poll interval (relaxed to ~10 min when a + // webhook is live) instead of probing every ~60s tick. + let interval_s = auto_pull.effective_poll_interval_s() as i64; + let poll_key = format!("{}|{}", row.workspace_id, repo.git_repo_resource_path); + let now = chrono::Utc::now().timestamp(); + { + let mut last = AUTO_PULL_LAST_POLL.lock().unwrap(); + if let Some(&t) = last.get(&poll_key) { + if now - t < interval_s - AUTO_PULL_POLL_SLACK_S { + continue; + } + } + last.insert(poll_key, now); + } + + let head = match windmill_store::resources::get_git_repo_head_for_autopull( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + ) + .await + { + Ok(Some(h)) => Ok(Some(h)), + // App-backed repos store a tokenless URL, so the ls-remote head + // check can't authenticate and returns None. Poll the head over + // the GitHub API with a minted installation token instead. This + // is the polling fallback/safety-net for auto- and polling-mode + // app repos whose webhook isn't live (unreachable instance, + // missing permission, or a dropped delivery). `webhook`-mode + // repos are skipped above and stay webhook-only. + Ok(None) => { + #[cfg(feature = "enterprise")] + { + windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + ) + .await + } + #[cfg(not(feature = "enterprise"))] + { + Ok(None) + } + } + Err(e) => Err(e), + }; + + match head { + Ok(Some((git_ref, sha))) => { + // Shared reconcile (also used by the webhook receiver): + // checks should_pull, enqueues, and records status/failure. + if let Err(e) = windmill_git_sync::reconcile_and_enqueue_pull( + db, + &row.workspace_id, + repo, + &git_ref, + &sha, + None, + ) + .await + { + tracing::warn!( + "git auto-pull: reconcile failed for {}/{}: {e:#}", + row.workspace_id, + repo.git_repo_resource_path + ); + } + + // Parent-managed fork sync: list the fork branches' heads in + // the same poll tick (one extra ls-remote / API call) and + // route each into its fork workspace. Needs the concrete + // tracked branch name to scope `wm-fork//*`; a + // branch-less resource resolves its default branch via + // `ls-remote --symref`, so "HEAD" only remains when that + // resolution failed. + if auto_pull.sync_forks && git_ref != "HEAD" { + poll_git_fork_branches( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + &git_ref, + ) + .await; + } + } + Ok(None) => {} + Err(e) => { + windmill_git_sync::record_auto_pull_failure( + db, + &row.workspace_id, + &repo.git_repo_resource_path, + &auto_pull.last_synced_sha, + format!("head check failed: {e}"), + ) + .await; + } + } + } + } + + Ok(()) +} + +/// Poll-side half of parent-managed fork sync (`sync_forks`): list every +/// `wm-fork//*` head of the parent's repo and reconcile each into +/// its fork workspace. Failures are logged, not recorded in the parent's pull +/// status — the parent's own sync state is unaffected by a fork's. +#[cfg(feature = "private")] +async fn poll_git_fork_branches( + db: &Pool, + parent_w_id: &str, + repo_path: &str, + base_branch: &str, +) { + // Dev-workspace children sync with their environment-label branch (`dev`/ + // `staging`) rather than the `wm-fork/**` pattern, so their branches must be + // listed explicitly. A label equal to the tracked branch is excluded — the + // parent's own head check covers it. + let label_refs: Vec = match sqlx::query_scalar!( + r#"SELECT DISTINCT COALESCE(dev_workspace_label, 'dev') as "label!" + FROM workspace + WHERE parent_workspace_id = $1 AND is_dev_workspace AND NOT deleted"#, + parent_w_id + ) + .fetch_all(db) + .await + { + Ok(labels) => labels.into_iter().filter(|l| l != base_branch).collect(), + Err(e) => { + tracing::warn!( + "git fork sync: failed to list dev-workspace labels for {parent_w_id}: {e:#}" + ); + Vec::new() + } + }; + + let fork_heads = match windmill_store::resources::get_git_repo_fork_heads_for_autopull( + db, + parent_w_id, + repo_path, + base_branch, + &label_refs, + ) + .await + { + Ok(Some(heads)) => Ok(heads), + // App-backed repos list fork refs over the GitHub API, mirroring the + // parent head check's fallback. + Ok(None) => { + #[cfg(feature = "enterprise")] + { + windmill_common::git_sync_ee::get_app_repo_fork_heads_for_autopull( + db, + parent_w_id, + repo_path, + base_branch, + &label_refs, + ) + .await + .map(|heads| heads.unwrap_or_default()) + } + #[cfg(not(feature = "enterprise"))] + { + Ok(Vec::new()) + } + } + Err(e) => Err(e), + }; + match fork_heads { + Ok(heads) => { + for (branch, sha) in heads { + if let Err(e) = windmill_git_sync::reconcile_fork_branch_pull( + db, + parent_w_id, + repo_path, + &branch, + base_branch, + &sha, + ) + .await + { + tracing::warn!( + "git fork sync: reconcile failed for {parent_w_id}/{repo_path} branch {branch}: {e:#}" + ); + } + } + } + Err(e) => { + tracing::warn!( + "git fork sync: fork branch listing failed for {parent_w_id}/{repo_path}: {e:#}" + ); + } + } +} + async fn vacuuming_tables(db: &Pool) -> error::Result<()> { sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") .execute(db) @@ -4183,12 +4872,95 @@ WHERE concurrency_id IN (SELECT concurrency_id FROM rows_to_delete) RETURNING c Ok(()) } +/// Memory usage at a worker's last ping as a fraction of its cgroup limit. +/// Takes the larger of the cgroup-wide reading and the windmill process's +/// jemalloc resident — if only one is present, that value wins; if both are +/// present, the larger is the more conservative (higher-signal) choice. +fn zombie_worker_memory_pct( + usage: Option, + wm_usage: Option, + total: Option, +) -> Option { + let total = total?; + if total <= 0 { + return None; + } + let used = usage.max(wm_usage)?; + Some(used as f64 / total as f64) +} + +struct ZombieFlowCulprit { + worker: String, + ping_at: DateTime, + memory_usage: Option, + wm_memory_usage: Option, + memory_total: Option, + worker_group: Option, + worker_instance: Option, + ping_delta_secs: Option, +} + +/// Finds the worker that likely performed and dropped the flow's final state +/// transition when `q.worker` (the outer queue-row worker) looks healthy — a +/// different worker on the same pod/group whose *latest* ping is frozen in the +/// `[last_ping-5s, +15s]` window (a live worker would have advanced its in-place +/// ping past that old window, so a frozen ping there proves it has gone silent), +/// nearest the transition. Diagnostics-only: fails soft to `None`. +async fn find_zombie_flow_culprit_worker( + db: &DB, + q_worker: &str, + last_ping: DateTime, +) -> Option { + let res = sqlx::query_as!( + ZombieFlowCulprit, + r#" + WITH ref AS ( + SELECT worker_instance, worker_group FROM worker_ping WHERE worker = $1 LIMIT 1 + ) + SELECT + wp.worker AS "worker!", + wp.ping_at AS "ping_at!", + wp.memory_usage, + wp.wm_memory_usage, + wp.memory AS memory_total, + wp.worker_group, + wp.worker_instance, + EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))::float8 AS ping_delta_secs + FROM worker_ping wp, ref + WHERE wp.worker <> $1 + AND ( + (ref.worker_instance IS NOT NULL AND wp.worker_instance = ref.worker_instance) + OR (ref.worker_group IS NOT NULL AND wp.worker_group = ref.worker_group) + ) + AND wp.ping_at >= $2::timestamptz - interval '5 seconds' + AND wp.ping_at <= $2::timestamptz + interval '15 seconds' + ORDER BY ABS(EXTRACT(EPOCH FROM (wp.ping_at - $2::timestamptz))) ASC + LIMIT 1 + "#, + q_worker, + last_ping, + ) + .fetch_optional(db) + .await; + match res { + Ok(culprit) => culprit, + Err(e) => { + tracing::warn!( + "failed to query for zombie-flow culprit worker (q_worker={q_worker}): {e:#}" + ); + None + } + } +} + async fn handle_zombie_flows(db: &DB) -> error::Result<()> { + // flow_status is cast ::text on purpose: decoding the jsonb column directly as Box + // yields its binary form (leading version byte) and fails serde_json parsing at column 1. let flows = sqlx::query!( r#" SELECT j.id AS "id!", j.workspace_id AS "workspace_id!", j.parent_job, j.flow_step_id IS NOT NULL AS "is_flow_step?", - COALESCE(s.flow_status, s.workflow_as_code_status) AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?", + COALESCE(s.flow_status, s.workflow_as_code_status)::text AS "flow_status: Box", r.ping AS last_ping, j.same_worker AS "same_worker?", q.worker AS "worker?", wp.ping_at AS "worker_last_ping?", wp.memory_usage AS "worker_memory_usage?", @@ -4217,11 +4989,7 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { .as_deref() .and_then(|x| serde_json::from_str::(x).ok()); if !flow.same_worker.unwrap_or(false) - && status.is_some_and(|s| { - s.modules - .get(0) - .is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. })) - }) + && status.as_ref().is_some_and(|s| s.is_not_yet_started()) { let error_message = format!( "Zombie flow detected: {} in workspace {}. It hasn't started yet, restarting it.", @@ -4282,18 +5050,35 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { // worker name; this flow's recorded worker name still points at the // dead process whose last ping can be under 60s old, and the memory // signal is what lets us catch that window. - let memory_pct: Option = flow.worker_memory_total.and_then(|total| { - if total <= 0 { - return None; - } - let used = flow.worker_memory_usage.max(flow.worker_wm_memory_usage)?; - Some(used as f64 / total as f64) - }); + let memory_pct: Option = zombie_worker_memory_pct( + flow.worker_memory_usage, + flow.worker_wm_memory_usage, + flow.worker_memory_total, + ); let oom_strong = memory_pct.is_some_and(|p| p >= 0.85); let oom_moderate = memory_pct.is_some_and(|p| p >= 0.60); let mem_pct_str = memory_pct .map(|p| format!("{:.1}% of container limit", (p * 100.0).min(100.0))) .unwrap_or_else(|| "memory unknown at last ping".to_string()); + + // When q.worker itself already shows OOM evidence the diagnosis below is + // already correct. Otherwise q.worker is likely a bystander (the outer + // queue-row worker) and the dropped transition was performed by a + // different worker on the same pod/group that OOM-died — go find it. + let q_worker_shows_oom = oom_moderate || worker_ping_stale == Some(true); + let culprit = if q_worker_shows_oom { + None + } else if let (Some(qw), Some(lp)) = (flow.worker.as_deref(), flow.last_ping) { + find_zombie_flow_culprit_worker(db, qw, lp).await + } else { + None + }; + let culprit_pct = culprit.as_ref().and_then(|c| { + zombie_worker_memory_pct(c.memory_usage, c.wm_memory_usage, c.memory_total) + }); + let culprit_pct_str = + culprit_pct.map(|p| format!("{:.1}% of its memory limit", (p * 100.0).min(100.0))); + let worker_info = if let Some(worker_name) = flow.worker.as_deref() { let mut s = format!("\nWorker handling the flow: {worker_name}"); match (flow.worker_group.as_deref(), flow.worker_version.as_deref()) { @@ -4324,8 +5109,11 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { (false, false, true) => format!( "LIKELY OOM-KILLED — {mem_pct_str} at last ping (a replacement worker process may have started in the same pod under a new windmill worker name)" ), + (false, false, false) if culprit.is_some() => { + "still pinging with healthy memory — this is NOT the worker that performed the flow's final state transition (see likely culprit worker below)".to_string() + } (false, false, false) => { - "worker still pinging with healthy memory — likely deadlocked or blocking on the state transition".to_string() + "worker still pinging with healthy memory — most likely a different worker performed and dropped the final transition (see hint); less likely: this worker deadlocked or is blocking on the state transition".to_string() } }; s.push_str(&format!("\nWorker last ping: {wp} ({age}s ago) — {status}")); @@ -4370,20 +5158,88 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { .to_string() }; - let hint: String = match (worker_ping_stale, oom_moderate) { - (Some(_), true) => format!( - "\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)." - ), - (Some(true), false) => { - "\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string() - } - (Some(false), false) => { - "\nWorker is still pinging and memory looked healthy at its last ping — most likely a deadlock or blocking call during the state transition. Capture a stack trace (e.g. via SIGQUIT) from the worker process. As a sanity check, also verify pod restart count in case a replacement worker process in the same pod has silently taken over.".to_string() - } - (None, _) => String::new(), + let culprit_info = if let Some(c) = culprit.as_ref() { + let age = (now - c.ping_at).num_seconds(); + let rel = match c.ping_delta_secs { + Some(d) if d >= 0.0 => format!("{d:.0}s after"), + Some(d) => format!("{:.0}s before", -d), + None => "around".to_string(), + }; + let loc = + if c.worker_instance.is_some() && c.worker_instance == flow.worker_instance { + format!( + "same pod/instance '{}'", + c.worker_instance.as_deref().unwrap() + ) + } else if let Some(g) = c.worker_group.as_deref() { + format!("worker group '{g}'") + } else if let Some(inst) = c.worker_instance.as_deref() { + format!("instance '{inst}'") + } else { + "same pod/group".to_string() + }; + let mem = match culprit_pct_str.as_deref() { + Some(p) => format!("was at {p}"), + None => "did not report memory".to_string(), + }; + let mem_detail = match (c.memory_usage, c.wm_memory_usage, c.memory_total) { + (host, wm, Some(total)) => { + let used = host.max(wm); + match used { + Some(u) => format!( + " (memory at last ping: {} of {})", + fmt_mb(u), + fmt_mb(total) + ), + None => format!(" (total available: {})", fmt_mb(total)), + } + } + _ => String::new(), + }; + let q_name = flow.worker.as_deref().unwrap_or("the recorded worker"); + format!( + "\nLikely culprit worker (on {loc}): {} — last pinged {} ({age}s ago, {rel} this flow's last ping) and then stopped pinging; it {mem} at that last ping{mem_detail}. This flow's dropped state transition was most likely performed by this worker and lost to its death (most likely OOM-kill), not a deadlock on {q_name}.", + c.worker, c.ping_at, + ) + } else { + String::new() }; - let service_logs_info = match (flow.worker_instance.as_deref(), flow.worker_last_ping) { + let hint: String = if let Some(c) = culprit.as_ref() { + let q_name = flow.worker.as_deref().unwrap_or("the recorded worker"); + match culprit_pct { + Some(p) if p >= 0.60 => format!( + "\nThis is almost certainly an OOM-kill on a *different* worker: {} (on the same pod/worker group) was at {} at its last ping right around this flow's transition, then stopped pinging. The flow's recorded worker ({q_name}) looks healthy because it is not the worker that performed the dropped transition. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`).", + c.worker, + culprit_pct_str.as_deref().unwrap_or("a high fraction of its limit"), + ), + _ => format!( + "\nMost likely an OOM-kill on a *different* worker: {} (on the same pod/worker group) stopped pinging right around this flow's transition ({last_ping:?}); its memory may have spiked after its last ping or not been reported. The flow's recorded worker ({q_name}) looks healthy because it did not perform the dropped transition. First check that pod's restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Only if that worker was not OOM-killed, consider a deadlock on {q_name} and capture a stack trace (e.g. via SIGQUIT).", + c.worker, + ), + } + } else { + match (worker_ping_stale, oom_moderate) { + (Some(_), true) => format!( + "\nThis is almost certainly an OOM-kill: container memory at the worker's last ping was at {mem_pct_str}. Raise the worker memory limit (e.g. k8s `resources.limits.memory`) or reduce per-flow memory usage. Confirm via pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`)." + ), + (Some(true), false) => { + "\nWorker stopped pinging and its last memory snapshot did not look high — in practice the overwhelmingly common cause here is still OOM-kill (memory may have spiked between the last ping and the kill, or never been reported). First check pod restart count (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`). Less likely: host failure, network partition, or a panic — check worker logs / k8s events around the last ping time.".to_string() + } + (Some(false), false) => { + format!("\nThe flow's recorded worker is still pinging with healthy memory, but that worker is often NOT the one that performed the final state transition (in nested/subflow/forloop cases the last iteration runs on another worker). First check whether a different worker on the same pod / worker group was OOM-killed around {last_ping:?} (`kubectl describe pod` / `kube_pod_container_status_last_terminated_reason`, and that pod's worker memory metrics). Only if no such worker died, treat this as a deadlock or blocking call on the recorded worker during the state transition and capture a stack trace (e.g. via SIGQUIT).") + } + (None, _) => String::new(), + } + }; + + // Pull logs for the worker (and around the time) we actually blame: the + // culprit's instance/last-ping when one was found, else q.worker's. + let (log_instance, log_ping) = match culprit.as_ref() { + Some(c) => (c.worker_instance.as_deref(), Some(c.ping_at)), + None => (flow.worker_instance.as_deref(), flow.worker_last_ping), + }; + let service_logs_info = match (log_instance, log_ping) { (Some(host), Some(wlp)) => { let after = (wlp - chrono::Duration::seconds(90)) .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); @@ -4437,13 +5293,25 @@ async fn handle_zombie_flows(db: &DB) -> error::Result<()> { }; let reason = format!( - "{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{hint}{service_logs_info}", + "{} was hanging in between 2 steps. Last ping: {last_ping:?} (now: {now}){worker_info}{culprit_info}{hint}{service_logs_info}", if flow.is_flow_step.unwrap_or(false) && flow.parent_job.is_some() { format!("Flow was cancelled because subflow {id} ({base_url}/run/{id}?workspace={workspace_id})") } else { format!("Flow {id} ({base_url}/run/{id}?workspace={workspace_id}) was cancelled because it") } ); + let reason = match between_steps_recovery_guidance( + db, + status.as_ref(), + id, + &workspace_id, + &base_url, + ) + .await + { + Some(guidance) => format!("{reason}\n\n{guidance}"), + None => reason, + }; report_critical_error(reason.clone(), db.clone(), Some(&flow.workspace_id), None).await; cancel_zombie_flow_job(db, flow.id, &flow.workspace_id, format!(r#"{reason} @@ -4490,6 +5358,103 @@ Please check your worker logs for more details and feel free to report it to the Ok(()) } +/// When a between-steps zombie's stuck step has every child recorded as a +/// `success` completion, the flow's state is fully derivable: only the final +/// state transition was lost to the worker failure, not any real work. In that +/// case return concrete restart-from-step recovery guidance to append to the +/// cancellation reason / critical alert. Returns `None` when the state isn't +/// derivable (some child missing or not successful), so the existing wording is +/// left untouched. Auto-recovery is deliberately not attempted (a re-driven +/// transition can OOM again on the same aggregated state; a human raises the +/// memory limit first, then restarts). +async fn between_steps_recovery_guidance( + db: &DB, + status: Option<&FlowStatus>, + flow_id: Uuid, + workspace_id: &str, + base_url: &str, +) -> Option { + // The stuck module is the current step, left InProgress because the + // transition that would have marked it Success was dropped. It is only + // derivable when its own cursor reached the end (a serial fan-out reaped + // mid-iteration has unrun work left; while-loops are never derivable). Whether + // restart reuses the children or re-runs the step (final step, or one carrying a + // stop/skip/approval/sleep) is decided by the restart path against the flow + // definition, which the reaper doesn't load; the guidance states both outcomes + // rather than promising reuse the restart might decline. + let status = status?; + let idx = usize::try_from(status.step).ok()?; + let module = status.modules.get(idx)?; + if !module.is_between_steps_complete() { + return None; + } + let step_id = module.id(); + + // Only a top-level deployed flow exposes a working restart-from-step: the run page's + // "Re-start from" button is rendered only for job_kind == 'flow' (a flowpreview, even a + // pathful editor preview, or a singlestepflow does not qualify), and a subflow child + // restarts via its root. Match that surface exactly so the guidance never points at a + // button / endpoint that isn't there; leave the existing wording otherwise. + let restartable = sqlx::query_scalar!( + r#"SELECT (kind = 'flow' AND parent_job IS NULL) AS "restartable!" + FROM v2_job WHERE id = $1"#, + flow_id, + ) + .fetch_one(db) + .await + .ok()?; + if !restartable { + return None; + } + + // Children whose completion the lost transition would have aggregated: the + // loop/branchall iterations, or the single leaf/subflow child. + let child_ids: Vec = module + .flow_jobs() + .filter(|v| !v.is_empty()) + .or_else(|| module.job().map(|j| vec![j]))?; + + // Derivable only when every child is recorded as a success completion. + let success_children = sqlx::query_scalar!( + "SELECT count(*) FROM v2_job_completed + WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'", + workspace_id, + &child_ids, + ) + .fetch_one(db) + .await + .ok()? + .unwrap_or(0); + if success_children != child_ids.len() as i64 { + return None; + } + let n = child_ids.len(); + + // For loop/branchall, name the completed iteration/branch count so the + // operator can confirm the whole fan-out is intact. + let iteration_hint = match module { + FlowStatusModule::InProgress { iterator: Some(_), .. } => { + format!(" (loop step, all {n} iterations completed)") + } + FlowStatusModule::InProgress { branchall: Some(_), .. } => { + format!(" (branchall step, all {n} branches completed)") + } + _ => String::new(), + }; + + Some(format!( + "RECOVERY: all {n} child job(s) of step `{step_id}`{iteration_hint} completed successfully; \ +only the flow's final state transition was lost to the worker failure above (not any genuine failure), so \ +the completed work is intact. To recover: first change the failure condition (raise the worker memory limit, \ +e.g. k8s `resources.limits.memory`, or move the flow to a larger worker group), then restart from step \ +`{step_id}`. Restart replays only the dropped transition and reuses the completed children where the step's \ +result is fully derivable; a step that is the flow's last, or carries a stop/skip condition, an approval, or a \ +sleep, is re-run instead (re-evaluating those on the larger worker).\n\ + UI: open {base_url}/run/{flow_id}?workspace={workspace_id} and use \"Re-start from {step_id}\".\n\ + API: POST {base_url}/api/w/{workspace_id}/jobs/restart/f/{flow_id} with body {{\"step_id\":\"{step_id}\"}}." + )) +} + async fn cancel_zombie_flow_job( db: &Pool, id: Uuid, @@ -5064,3 +6029,82 @@ mod retention_overrides_tests { assert!(parse_retention_overrides(over_cap).is_err()); } } + +#[cfg(test)] +mod strike_unarmed_tests { + use super::strike_unarmed; + use std::collections::{HashMap, HashSet}; + + fn key(path: &str) -> (String, String) { + ("ws".to_string(), path.to_string()) + } + + fn set(paths: &[&str]) -> HashSet<(String, String)> { + paths.iter().map(|p| key(p)).collect() + } + + /// The strike threshold is the only thing keeping the reconciler from racing + /// an in-flight push: `push_scheduled_job`'s own `already_exists` guard keys + /// on the same columns as the scan, so it is false by construction whenever a + /// schedule is found unarmed. + #[test] + fn rearms_only_after_consecutive_unarmed_passes() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a"])), vec![key("a")]); + } + + #[test] + fn armed_observation_resets_the_tally() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + // `a` is armed again on this pass, so its strike must not carry over. + assert!(strike_unarmed(&mut seen, set(&[])).is_empty()); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a"])), vec![key("a")]); + } + + #[test] + fn tallies_are_per_schedule() { + let mut seen = HashMap::new(); + assert!(strike_unarmed(&mut seen, set(&["a"])).is_empty()); + assert_eq!(strike_unarmed(&mut seen, set(&["a", "b"])), vec![key("a")]); + assert_eq!(strike_unarmed(&mut seen, set(&["b"])), vec![key("b")]); + } +} + +#[cfg(test)] +mod zombie_worker_memory_pct_tests { + use super::zombie_worker_memory_pct; + + #[test] + fn takes_the_larger_of_the_two_readings() { + // Both present: the larger (higher-signal) reading wins, not either + // one unconditionally — a "simplify to `usage.or(wm_usage)`" refactor + // would silently under-report and miss OOMs. + let p = zombie_worker_memory_pct(Some(600), Some(900), Some(1000)).unwrap(); + assert!((p - 0.9).abs() < f64::EPSILON); + } + + #[test] + fn falls_back_to_whichever_reading_is_present() { + assert_eq!( + zombie_worker_memory_pct(Some(700), None, Some(1000)), + Some(0.7) + ); + assert_eq!( + zombie_worker_memory_pct(None, Some(800), Some(1000)), + Some(0.8) + ); + } + + #[test] + fn none_when_no_usage_or_no_valid_total() { + assert_eq!(zombie_worker_memory_pct(None, None, Some(1000)), None); + assert_eq!(zombie_worker_memory_pct(Some(500), Some(500), None), None); + assert_eq!( + zombie_worker_memory_pct(Some(500), Some(500), Some(0)), + None + ); + } +} diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index abd39814b6..ab6cb7e286 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -101,7 +101,7 @@ group_: workspace_id(char), name(char), summary(text), extra_perms(jsonb) FK: (workspace_id) -> workspace(id) group_permission_history: id(bigint), workspace_id(char), group_name(char), changed_by(char), changed_at(ts), change_type(char), member_affected(char) FK: (workspace_id, group_name) -> group_(workspace_id, name) -healthchecks: id(bigint), check_type(char), healthy(bool), created_at(ts) +healthchecks: id(bigint), check_type(text), healthy(bool), created_at(ts) http_trigger: path(char), route_path(char), route_path_key(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), authentication_method(authentication_method), http_method(http_method), static_asset_config(jsonb), is_static_website(bool), workspaced_route(bool), wrap_body(bool), raw_string(bool), authentication_resource_path(char), summary(char), description(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), request_type(request_type), mode(trigger_mode), labels(text[]) input: id(uuid), workspace_id(char), runnable_id(char), runnable_type(runnable_type), name(text), args(jsonb), created_at(ts), created_by(char), is_public(bool) FK: (workspace_id) -> workspace(id) diff --git a/backend/tests/app_s3_onbehalf.rs b/backend/tests/app_s3_onbehalf.rs index 2e1085edd3..7b233b505d 100644 --- a/backend/tests/app_s3_onbehalf.rs +++ b/backend/tests/app_s3_onbehalf.rs @@ -1,7 +1,9 @@ -//! Deployed-app S3 reads authorize on-behalf of the app author and are confined -//! to app provenance (declared keys or recent job outputs): a viewer cannot read -//! an arbitrary `file_key` as the author. Requires the `parquet` feature — the -//! real `apps_u/*` S3 handlers are gated on it. +//! Deployed-app S3 reads authorize on-behalf of the app author and are confined to +//! app provenance (declared keys or recent job outputs): an anonymous viewer cannot +//! read an arbitrary `file_key` as the author. A viewer on a full (unscoped) session +//! instead falls back to reading as THEMSELVES (bounded by their own S3 perms), so the +//! gate is exercised here through the anonymous identity it still fully protects. +//! Requires the `parquet` feature — the real `apps_u/*` S3 handlers are gated on it. //! //! `base` fixture: test-user (admin, SECRET_TOKEN); test-user-2 (non-admin, //! SECRET_TOKEN_2, no S3 folder permission). @@ -21,6 +23,19 @@ fn client() -> reqwest::Client { reqwest::Client::new() } +/// Mint an API token for test-user (admin) restricted to `scopes`. +async fn mint_scoped_token(port: u16, scopes: Vec<&str>) -> anyhow::Result { + let resp = authed( + client().post(format!("http://localhost:{port}/api/users/tokens/create")), + ADMIN_TOKEN, + ) + .json(&json!({ "label": "scoped", "scopes": scopes, "workspace_id": "test-workspace" })) + .send() + .await?; + assert_eq!(resp.status(), 201, "mint scoped token"); + Ok(resp.text().await?) +} + fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { builder.header("Authorization", format!("Bearer {}", token)) } @@ -50,62 +65,54 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: .await?; assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); - // GET an app-scoped S3 route as `token`. No workspace storage is configured, - // so a request that clears the provenance gate fails later at the storage - // lookup (or the CE OSS stub), never with "File restricted" — which is what - // lets these assertions distinguish "gate passed" from "gate denied". - let get = |route: &str, token: &'static str| { + // GET an app-scoped S3 route ANONYMOUSLY. Anonymous callers have no viewer + // identity to fall back to, so the provenance gate still fully applies to them + // (unlike logged-in viewers, who now read as themselves — see the union test). + // No workspace storage is configured, so a request that clears the gate fails + // later at the storage lookup (or the CE OSS stub), never with the denial + // message — which is what lets these assertions distinguish pass from deny. + let get = |route: &str| { let url = format!("{ws}/apps_u/{route}"); - authed(client().get(url), token).send() + client().get(url).send() }; - let denied = |body: &str| body.contains("File restricted"); + let denied = |body: &str| body.contains("is not accessible from this app"); - // download_s3_file: author-on-behalf allowed for the declared key, denied for - // a key the app never declared (the confused-deputy guard). - let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}"), USER_TOKEN) + // download_s3_file: allowed for the declared key, denied for a key the app never + // declared (the confused-deputy guard). + let body = get(&format!("download_s3_file/{APP}?s3={DECLARED}")) .await? .text() .await?; assert!(!denied(&body), "declared key must clear the gate: {body}"); - let body = get( - &format!("download_s3_file/{APP}?s3={NON_PROVENANCE}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("download_s3_file/{APP}?s3={NON_PROVENANCE}")) + .await? + .text() + .await?; assert!(denied(&body), "non-provenance key must be denied: {body}"); // load_table_count and load_csv_preview enforce the same gate. The preview's // numeric `limit`/`offset` must deserialize (regression: a flattened query // struct 400s on them under serde_urlencoded). - let body = get( - &format!("load_table_count/{APP}?file_key={DECLARED}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("load_table_count/{APP}?file_key={DECLARED}")) + .await? + .text() + .await?; assert!( !denied(&body), "table_count declared key must clear the gate: {body}" ); - let body = get( - &format!("load_table_count/{APP}?file_key={NON_PROVENANCE}"), - USER_TOKEN, - ) - .await? - .text() - .await?; + let body = get(&format!("load_table_count/{APP}?file_key={NON_PROVENANCE}")) + .await? + .text() + .await?; assert!( denied(&body), "table_count non-provenance key must be denied: {body}" ); - let resp = get( - &format!("load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0"), - USER_TOKEN, - ) + let resp = get(&format!( + "load_csv_preview/{APP}?file_key={DECLARED}&limit=5&offset=0" + )) .await?; let status = resp.status(); let body = resp.text().await?; @@ -116,23 +123,16 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: ); // load_file_preview: `read_bytes_from` / `read_bytes_length` are required. - let resp = get( - &format!("load_file_preview/{APP}?file_key={DECLARED}"), - USER_TOKEN, - ) - .await?; + let resp = get(&format!("load_file_preview/{APP}?file_key={DECLARED}")).await?; assert_eq!( resp.status(), 400, "file_preview without byte range must 400: {}", resp.text().await? ); - let body = get( - &format!( - "load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096" - ), - USER_TOKEN, - ) + let body = get(&format!( + "load_file_preview/{APP}?file_key={DECLARED}&read_bytes_from=0&read_bytes_length=4096" + )) .await? .text() .await?; @@ -144,6 +144,222 @@ async fn test_deployed_app_s3_onbehalf_provenance(db: Pool) -> anyhow: Ok(()) } +/// The viewer-perm union: a viewer on a full (unscoped) session is no longer hard-denied +/// by the provenance gate for a pre-existing file. It falls back to reading as ITSELF +/// (bounded by its own S3 perms downstream), while an anonymous caller (no identity) and +/// a scope-restricted token (can hit `apps_u/*` but not `job_helpers/*`, so the fallback +/// would be a new capability) both stay fully gated with the actionable denial. +#[sqlx::test(fixtures("base"))] +async fn test_deployed_app_s3_viewer_union(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP, + "summary": "s3 viewer union test", + "value": {}, + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "allowed_s3_keys": [{ "s3_path": DECLARED }] + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); + + let url = format!("{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}"); + + // Anonymous: still gated. The denial is the actionable message and echoes the key. + let body = client().get(&url).send().await?.text().await?; + assert!( + body.contains("is not accessible from this app"), + "anonymous viewer must stay gated with the actionable denial: {body}" + ); + assert!( + body.contains(NON_PROVENANCE), + "denial must echo the requested key: {body}" + ); + + // Logged-in viewer: no longer hard-denied — the gate delegates to reading as the + // viewer, so the request falls through to the storage read (no gate denial in + // EITHER the old or new form). No workspace storage is configured here, so it + // surfaces a downstream storage/OSS error, not a gate denial. + let body = authed(client().get(&url), USER_TOKEN) + .send() + .await? + .text() + .await?; + assert!( + !body.contains("is not accessible from this app") && !body.contains("File restricted"), + "logged-in viewer must delegate to its own read, not be gate-denied: {body}" + ); + + // Scope-restricted token: an `apps:read:` token reaches this route but is + // REJECTED by the route-scope middleware on `job_helpers/*`, so it must NOT get the + // viewer fallback (that would be a capability it cannot obtain directly). It stays + // gated with the denial, unlike the unscoped session above. + let apps_read_scope = format!("apps:read:{APP}"); + let scoped = mint_scoped_token(port, vec![apps_read_scope.as_str()]).await?; + let body = authed(client().get(&url), &scoped) + .send() + .await? + .text() + .await?; + assert!( + body.contains("is not accessible from this app"), + "scope-restricted token must stay gated, not get the viewer fallback: {body}" + ); + + // A filter-tags-only token carries no real scope restriction (the route-scope + // middleware treats it as unscoped), so it can read via job_helpers directly and + // MUST get the viewer fallback here — not be gated like a genuinely scoped token. + let tag_only = mint_scoped_token(port, vec!["if_jobs:filter_tags:default"]).await?; + let body = authed(client().get(&url), &tag_only) + .send() + .await? + .text() + .await?; + assert!( + !body.contains("is not accessible from this app") && !body.contains("File restricted"), + "filter-tags-only token is effectively unscoped and must delegate, not be gated: {body}" + ); + + Ok(()) +} + +/// Mint a presigned bearer (`exp=..&sig=..`) exactly as `sign_s3_objects` does: +/// `HMAC-SHA256(workspace_key, "file_key={s3}&exp={exp}")` (no storage param, since +/// these routes send none). `validate_s3_signature` is `private`-gated, so this test +/// only runs with the `private` feature. +#[cfg(feature = "private")] +fn mint_presigned(workspace_key: &str, s3: &str, exp: i64) -> String { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let mut mac = Hmac::::new_from_slice(workspace_key.as_bytes()).unwrap(); + mac.update(format!("file_key={s3}&exp={exp}").as_bytes()); + let sig = hex::encode(mac.finalize().into_bytes()); + format!("exp={exp}&sig={sig}") +} + +/// A presigned S3 object (bearer minted by `signS3Objects`) bypasses the provenance +/// gate on EVERY app-scoped display route, not just the raw `download_s3_file` +/// download: a valid signature clears the gate on preview/count/metadata/csv routes, +/// while an unsigned key stays denied and a forged/expired signature is rejected. +#[cfg(feature = "private")] +#[sqlx::test(fixtures("base"))] +async fn test_deployed_app_s3_presigned_bypasses_gate(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + + let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP, + "summary": "s3 presigned test", + "value": {}, + "policy": { + "execution_mode": "anonymous", + "triggerables": {}, + "allowed_s3_keys": [{ "s3_path": DECLARED }] + } + })) + .send() + .await?; + assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); + + let workspace_key: String = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = 'test-workspace' AND kind = 'cloud'", + ) + .fetch_one(&db) + .await?; + let exp = chrono::Utc::now().timestamp() + 3600; + let presigned = mint_presigned(&workspace_key, NON_PROVENANCE, exp); + + let get = |route: String, token: &'static str| { + let url = format!("{ws}/apps_u/{route}"); + authed(client().get(url), token).send() + }; + let denied = |body: &str| body.contains("is not accessible from this app"); + + // Control: NON_PROVENANCE without a signature is denied by the gate. Sent + // anonymously — a logged-in viewer would instead fall back to reading as + // themselves, so anonymous is the identity that isolates the presigned bypass. + let body = client() + .get(format!( + "{ws}/apps_u/download_s3_file/{APP}?s3={NON_PROVENANCE}" + )) + .send() + .await? + .text() + .await?; + assert!( + denied(&body), + "unsigned non-provenance key must be denied: {body}" + ); + + // Every display route: a valid presigned key clears the gate (falls through to + // the storage read, which fails with a storage error, never "File restricted"). + // `read_bytes_*` are required on load_file_preview. + let routes = [ + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{presigned}"), + format!("load_table_count/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + format!("load_csv_preview/{APP}?file_key={NON_PROVENANCE}&limit=5&offset=0&{presigned}"), + format!("load_parquet_preview/{APP}?file_key={NON_PROVENANCE}&limit=5&offset=0&{presigned}"), + format!("load_file_metadata/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + format!( + "load_file_preview/{APP}?file_key={NON_PROVENANCE}&read_bytes_from=0&read_bytes_length=4096&{presigned}" + ), + format!("download_s3_parquet_file_as_csv/{APP}?file_key={NON_PROVENANCE}&{presigned}"), + ]; + for route in routes { + let body = get(route.clone(), USER_TOKEN).await?.text().await?; + assert!( + !denied(&body), + "presigned key must bypass the gate on {route}: {body}" + ); + } + + // A tampered signature must NOT bypass: presence of `sig` commits to validation, + // so a wrong sig is rejected outright ("Invalid signature") rather than falling + // back to the provenance gate. + let forged = format!("exp={exp}&sig={}", "00".repeat(32)); + let body = get( + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{forged}"), + USER_TOKEN, + ) + .await? + .text() + .await?; + assert!( + body.contains("Invalid signature"), + "forged signature must be rejected: {body}" + ); + + // An expired-but-valid signature is rejected on expiry, not bypassed. + let past = chrono::Utc::now().timestamp() - 10; + let expired = mint_presigned(&workspace_key, NON_PROVENANCE, past); + let body = get( + format!("download_s3_file/{APP}?s3={NON_PROVENANCE}&{expired}"), + USER_TOKEN, + ) + .await? + .text() + .await?; + assert!( + body.contains("Signature expired"), + "expired signature must be rejected: {body}" + ); + + Ok(()) +} + /// Seed a completed job whose result carries an s3 object. `app_trigger` sets the /// app-origination marker exactly as `execute_component` stamps it: `Some(app_path)` /// => `trigger_kind = 'app'` + `trigger = ` (an app-launched run); @@ -179,12 +395,14 @@ async fn seed_completed_job( } /// A deployed app that renders S3 files it produced (e.g. a SQL query persisted to -/// S3 by a component) must clear the provenance gate for the viewer whose own app -/// run produced them, while (a) a viewer cannot forge provenance by running a -/// runnable directly (no app marker), (b) another app's outputs stay denied, and -/// (c) another viewer's outputs stay denied (cross-viewer isolation). Provenance is -/// keyed on the app-origination marker (`trigger_kind='app'` + `trigger=`) -/// that `execute_component` stamps, plus `created_by = ` for isolation. +/// S3 by a component) must clear the provenance gate for the caller whose own app run +/// produced them, while (a) provenance cannot be forged by running a runnable directly +/// (no app marker), (b) another app's outputs stay denied, and (c) another caller's +/// outputs stay denied (per-caller isolation). Provenance is keyed on the +/// app-origination marker (`trigger_kind='app'` + `trigger=`) that +/// `execute_component` stamps, plus `created_by = ` for isolation. +/// Exercised anonymously: the gate still fully governs anonymous callers, whereas a +/// logged-in viewer would instead fall back to reading as themselves. #[sqlx::test(fixtures("base"))] async fn test_deployed_app_s3_onbehalf_flow_script_provenance( db: Pool, @@ -197,10 +415,10 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( const FS_APP: &str = "u/test-user/s3flowscript"; const OTHER_APP: &str = "u/test-user/other_app"; - // Produced by test-user-2's own app run of THIS app. - const USER_KEY: &str = "results/user2_output.parquet"; - // Produced by test-user's own app run of THIS app. - const ADMIN_KEY: &str = "results/admin_output.parquet"; + // Produced by the anonymous caller's own app run of THIS app. + const OWN_KEY: &str = "results/own_output.parquet"; + // Produced by a DIFFERENT caller's app run of THIS app → isolation, must stay denied. + const OTHER_CALLER_KEY: &str = "results/user2_output.parquet"; // Produced by an app run of a DIFFERENT app → must stay denied. const OTHER_APP_KEY: &str = "results/other_app_output.parquet"; // Produced by a DIRECT run (no app marker) → the forgery attempt, must stay denied. @@ -217,64 +435,49 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( .await?; assert_eq!(resp.status(), 201, "app create: {}", resp.text().await?); - // Seed the produced-file jobs (all within the 3h window). - seed_completed_job(&db, "test-user-2", Some(FS_APP), USER_KEY).await?; - seed_completed_job(&db, "test-user", Some(FS_APP), ADMIN_KEY).await?; - seed_completed_job(&db, "test-user-2", Some(OTHER_APP), OTHER_APP_KEY).await?; - seed_completed_job(&db, "test-user-2", None, FORGED_KEY).await?; + // Seed the produced-file jobs (all within the 3h window). The gate's `created_by` + // filter uses "anonymous" for an unauthenticated caller. + seed_completed_job(&db, "anonymous", Some(FS_APP), OWN_KEY).await?; + seed_completed_job(&db, "test-user-2", Some(FS_APP), OTHER_CALLER_KEY).await?; + seed_completed_job(&db, "anonymous", Some(OTHER_APP), OTHER_APP_KEY).await?; + seed_completed_job(&db, "anonymous", None, FORGED_KEY).await?; - let get = |route: &str, token: &'static str| { + let denied = |body: &str| body.contains("is not accessible from this app"); + // Anonymous GET (borrows `ws`, reusable across calls: the URL is built before the + // `async move` so only the owned `url` is moved into the future, not `ws`). + let anon_body = |route: String| { let url = format!("{ws}/apps_u/{route}"); - authed(client().get(url), token).send() - }; - let denied = |body: &str| body.contains("File restricted"); - let body_of = |route: String, token: &'static str| async move { - get(&route, token).await.unwrap().text().await.unwrap() + async move { + client() + .get(url) + .send() + .await + .unwrap() + .text() + .await + .unwrap() + } }; - // The viewer's own app run's output clears the gate (the case that regressed to - // "File restricted"). - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={USER_KEY}"), - USER_TOKEN, - ) - .await; + // The caller's own app run's output clears the gate (the case that regressed to + // a hard denial). + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OWN_KEY}")).await; assert!( !denied(&body), - "viewer's own app-produced key must clear the gate: {body}" + "caller's own app-produced key must clear the gate: {body}" ); - // The admin viewer's own app run's output clears — the gate has no admin bypass, - // it just matches the caller's own runs. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={ADMIN_KEY}"), - ADMIN_TOKEN, - ) - .await; - assert!( - !denied(&body), - "admin's own app-produced key must clear the gate: {body}" - ); - - // Cross-viewer isolation: the admin cannot pull test-user-2's result even though - // it is a genuine app-marked job of the same app (no admin bypass either). - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={USER_KEY}"), - ADMIN_TOKEN, - ) - .await; + // Per-caller isolation: another caller's result stays denied even though it is a + // genuine app-marked job of the same app. + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_CALLER_KEY}")).await; assert!( denied(&body), - "another viewer's app-produced key must stay denied (isolation): {body}" + "another caller's app-produced key must stay denied (isolation): {body}" ); // A key produced by a direct run (no app marker) stays denied — the forgery the // app-origination marker closes. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}"), - USER_TOKEN, - ) - .await; + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={FORGED_KEY}")).await; assert!( denied(&body), "key from a direct run (no app marker) must stay denied: {body}" @@ -282,11 +485,7 @@ async fn test_deployed_app_s3_onbehalf_flow_script_provenance( // A key produced by a DIFFERENT app stays denied — provenance is scoped to THIS // app's path. - let body = body_of( - format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}"), - USER_TOKEN, - ) - .await; + let body = anon_body(format!("download_s3_file/{FS_APP}?s3={OTHER_APP_KEY}")).await; assert!( denied(&body), "key produced by a different app must stay denied: {body}" diff --git a/backend/tests/fixtures/wac_approval_urls.sql b/backend/tests/fixtures/wac_approval_urls.sql new file mode 100644 index 0000000000..099b185e30 --- /dev/null +++ b/backend/tests/fixtures/wac_approval_urls.sql @@ -0,0 +1,23 @@ +-- A Workflow-as-Code job in the queue, suspended on a wait_for_approval step +-- (see tests/wac_approval_urls.rs). WAC parents are plain script jobs with no +-- parent_job, which is what makes get_flow_info_for_resume treat them as WAC. +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, script_lang, runnable_path, tag, visible_to_owner +) VALUES ( + 'a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1', 'test-workspace', 'test-user', + '2023-01-01 00:00:00', 'u/test-user', 'test@windmill.dev', + 'script', 'bun', 'u/test-user/wac_workflow', 'bun', true +); +INSERT INTO public.v2_job_queue (id, workspace_id, scheduled_for, running, suspend, tag) VALUES + ('a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1', 'test-workspace', '2023-01-01 00:00:00', true, 1, 'bun'); + +-- A second workspace the caller also administers, so a cross-workspace mint is +-- rejected by the job's workspace check rather than by workspace authorization. +INSERT INTO workspace (id, name, owner) VALUES + ('test-workspace-2', 'test-workspace-2', 'test-user'); +INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES + ('test-workspace-2', 'test@windmill.dev', 'test-user', true, 'Admin'); +INSERT INTO workspace_key(workspace_id, kind, key) VALUES + ('test-workspace-2', 'cloud', 'test-key-2'); +INSERT INTO workspace_settings (workspace_id) VALUES ('test-workspace-2'); diff --git a/backend/tests/nativets_jobs.rs b/backend/tests/nativets_jobs.rs index 610d0c5ea8..ff871e2da6 100644 --- a/backend/tests/nativets_jobs.rs +++ b/backend/tests/nativets_jobs.rs @@ -263,6 +263,43 @@ export function main(): number[] { ); } + // -- result + wm_labels carrying a NUL: must complete, jsonb-safe & text[]-safe -- + { + // `\u0000` here is 6 literal chars in the raw string; the JS runtime emits + // a real U+0000. It aborts the jsonb result insert (22P05) and, via + // wm_labels, the `text[]` labels update - both in the completion tx. + // `nul`/label come back stripped; `literal` (escaped backslash + text + // "u0000", no real NUL) survives untouched. + let result = push_and_wait( + &db, + RunJob::from(nativets_code( + r#"//native + +export function main(): {nul: string, literal: string, wm_labels: string[]} { + return { nul: "a\u0000b", literal: "a\\u0000b", wm_labels: ["x\u0000y"] }; +} +"#, + )), + &mut listener, + ) + .await; + assert!(result.success, "nul_result failed: {:?}", result.result); + let val = result.json_result().unwrap(); + assert_eq!(val["nul"], serde_json::json!("ab")); + assert_eq!(val["literal"], serde_json::json!("a\\u0000b")); + + // The wm_labels entry is persisted to the `text[]` column NUL-free. + let labels: Option> = + sqlx::query_scalar("SELECT labels FROM v2_job WHERE id = $1") + .bind(result.id) + .fetch_one(&db) + .await + .unwrap(); + let labels = labels.unwrap_or_default(); + assert!(labels.iter().any(|l| l == "xy"), "expected stripped label, got {labels:?}"); + assert!(!labels.iter().any(|l| l.contains('\0')), "labels must be NUL-free: {labels:?}"); + } + killpill.send(); Ok(()) } diff --git a/backend/tests/postgres_trigger_scope.rs b/backend/tests/postgres_trigger_scope.rs new file mode 100644 index 0000000000..1172e03ab6 --- /dev/null +++ b/backend/tests/postgres_trigger_scope.rs @@ -0,0 +1,69 @@ +//! Postgres-trigger ancillary handlers (slot / publication / version management) +//! must reject a path-mismatched scoped token before opening any connection — +//! the route-level middleware only checks the scope domain, so per-path +//! enforcement lives in the handlers. Rejecting pre-connection is why these tests +//! need no real Postgres resource. + +use axum::{extract::Path, Extension, Json}; +use sqlx::{Pool, Postgres}; +use windmill_api_auth::ApiAuthed; +use windmill_common::{db::UserDB, error::Error}; +use windmill_trigger_postgres::{handler, Slot}; + +fn scoped_authed(scopes: Vec<&str>) -> ApiAuthed { + ApiAuthed { + email: "alice@windmill.dev".to_string(), + username: "alice".to_string(), + is_admin: false, + is_operator: false, + groups: vec![], + folders: vec![], + scopes: Some(scopes.into_iter().map(str::to_string).collect()), + username_override: None, + token_prefix: None, + read_only: false, + } +} + +// A token scoped to `u/alice/db` must not reach a read handler for `u/bob/db`. +#[sqlx::test] +async fn read_handler_rejects_path_mismatched_scope(db: Pool) -> anyhow::Result<()> { + let authed = scoped_authed(vec!["postgres_triggers:read:u/alice/db"]); + let user_db = UserDB::new(db.clone()); + + let res = handler::get_postgres_version( + authed, + Extension(db), + Extension(user_db), + Path(("test-workspace".to_string(), "u/bob/db".to_string())), + ) + .await; + + assert!( + matches!(res, Err(Error::PermissionDenied(_))), + "expected PermissionDenied, got {res:?}" + ); + Ok(()) +} + +// The destructive slot-drop handler must reject a write token scoped to another path. +#[sqlx::test] +async fn drop_slot_rejects_path_mismatched_scope(db: Pool) -> anyhow::Result<()> { + let authed = scoped_authed(vec!["postgres_triggers:write:u/alice/db"]); + let user_db = UserDB::new(db.clone()); + + let res = handler::drop_slot_name( + authed, + Extension(user_db), + Extension(db), + Path(("test-workspace".to_string(), "u/bob/db".to_string())), + Json(Slot { name: "some_slot".to_string() }), + ) + .await; + + assert!( + matches!(res, Err(Error::PermissionDenied(_))), + "expected PermissionDenied, got {res:?}" + ); + Ok(()) +} diff --git a/backend/tests/runnables_list_pagination.rs b/backend/tests/runnables_list_pagination.rs new file mode 100644 index 0000000000..05655f4561 --- /dev/null +++ b/backend/tests/runnables_list_pagination.rs @@ -0,0 +1,417 @@ +//! Regression tests for the merged runnables listing endpoint +//! (`GET /w/{workspace}/runnables/list`), which UNION-ALLs scripts, flows and +//! apps into one keyset-paginated, globally-ordered stream. +//! +//! The delicate parts: +//! 1. Cross-kind ties — a script and a flow sharing the exact same +//! `(sort_key, path)` must each appear exactly once across pages, never +//! duplicated or skipped, in every order (the keyset cursor +//! `(sort_key, path, kind, tiebreak)`). +//! 2. Archived view semantics — scripts keep every version as a row (old ones +//! archived=true), so "Only archived" must key off the LATEST row per path: +//! an active path's superseded version must not leak in, and a fully-archived +//! path appears exactly once. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +fn new_script(path: &str, summary: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "description": "", + "content": "export async function main() {}", + "language": "deno", + "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": {}, "required": [] } + }) +} + +fn new_flow(path: &str, summary: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "description": "", + "value": { "modules": [] }, + "schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": {}, "required": [] } + }) +} + +/// Page through the endpoint one item at a time and return the ordered list of +/// `type:path` identifiers. +async fn paginate_all(port: u16, query: &str) -> Vec { + let base = format!("http://localhost:{port}/api/w/test-workspace/runnables/list"); + let mut out = vec![]; + let mut cursor: Option = None; + for _ in 0..50 { + let mut url = format!("{base}?{query}&per_page=1"); + if let Some(c) = &cursor { + url.push_str(&format!("&cursor={c}")); + } + let resp = authed(client().get(&url), "SECRET_TOKEN") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "list should succeed"); + let body: serde_json::Value = resp.json().await.unwrap(); + for it in body["items"].as_array().unwrap() { + out.push(format!( + "{}:{}", + it["type"].as_str().unwrap(), + it["path"].as_str().unwrap() + )); + } + match body["next_cursor"].as_str() { + Some(c) => cursor = Some(c.to_string()), + None => break, + } + } + out +} + +fn assert_no_dupes(items: &[String], label: &str) { + let mut sorted = items.to_vec(); + sorted.sort(); + let mut deduped = sorted.clone(); + deduped.dedup(); + assert_eq!( + sorted, deduped, + "{label}: keyset pagination must not duplicate or skip items, got {items:?}" + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_keyset_no_dupes_across_kind_ties(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + + // A script and a flow sharing a path, plus two more scripts. + for (p, s) in [ + ("u/test-user/tie", "Tie item"), + ("u/test-user/aaa", "Aaa"), + ("u/test-user/zzz", "Zzz"), + ] { + let r = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&new_script(p, s)) + .send() + .await?; + assert_eq!(r.status(), 201, "create script: {}", r.text().await?); + } + let r = authed( + client().post(format!("{base}/flows/create")), + "SECRET_TOKEN", + ) + .json(&new_flow("u/test-user/tie", "Tie item")) + .send() + .await?; + assert_eq!(r.status(), 201, "create flow: {}", r.text().await?); + + // Force an exact (sort_time, path, summary) tie between the script and flow + // at u/test-user/tie so the cross-kind boundary is actually exercised. + let ts = "2026-07-20 10:00:00+00"; + sqlx::query( + "UPDATE script SET created_at = $1::timestamptz WHERE workspace_id = 'test-workspace'", + ) + .bind(ts) + .execute(&db) + .await?; + sqlx::query( + "UPDATE flow SET edited_at = $1::timestamptz WHERE workspace_id = 'test-workspace'", + ) + .bind(ts) + .execute(&db) + .await?; + + // Expected: 3 scripts + 1 flow, each exactly once. + for q in [ + "order_by=updated&order_desc=true", + "order_by=name&order_desc=false", + "order_by=name&order_desc=true", + ] { + let items = paginate_all(port, q).await; + assert_eq!(items.len(), 4, "{q}: expected 4 items, got {items:?}"); + assert_no_dupes(&items, q); + assert!( + items.contains(&"flow:u/test-user/tie".to_string()), + "{q}: flow present" + ); + assert!( + items.contains(&"script:u/test-user/tie".to_string()), + "{q}: script present" + ); + } + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_archived_shows_only_paths_whose_latest_is_archived( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Scripts keep every version as its own row; superseded ones are archived=true. + // "Only archived" must key off the LATEST row per path, not every row. + + // Path A: an archived predecessor, then an active latest version. The path is + // active, so it must NOT appear in the archived view (the old version leaking in + // was the bug). + for (h, sec, archived) in [(811111111i64, 0, true), (822222222, 1, false)] { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, created_at, language, archived) + VALUES ('test-workspace', $1, 'u/test-user/active_hist', 'AH', '', 'x', 'test-user', ('2026-07-19 10:00:0' || $2)::timestamptz, 'deno', $3)", + ) + .bind(h).bind(sec.to_string()).bind(archived) + .execute(&db) + .await?; + } + + // Path B: fully archived — an older archived version plus a latest archived one. + // The archived view must surface it exactly once (its latest row), not per version. + for (h, sec) in [(833333333i64, 0), (844444444, 1)] { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, created_at, language, archived) + VALUES ('test-workspace', $1, 'u/test-user/archived_path', 'AP', '', 'x', 'test-user', ('2026-07-19 10:00:0' || $2)::timestamptz, 'deno', true)", + ) + .bind(h).bind(sec.to_string()) + .execute(&db) + .await?; + } + + // Favorite both paths (pinned first now); this test only checks presence/dedup, so + // pinning doesn't affect the assertions — favorite-ordering is covered separately. + for p in ["u/test-user/active_hist", "u/test-user/archived_path"] { + sqlx::query( + "INSERT INTO favorite (usr, workspace_id, path, favorite_kind) VALUES ('test-user', 'test-workspace', $1, 'script')", + ) + .bind(p) + .execute(&db) + .await?; + } + + // paginate_all pages one at a time, so a path crossing a page boundary is caught. + let items = paginate_all(port, "order_by=name&order_desc=false&show_archived=true").await; + assert_eq!( + items + .iter() + .filter(|i| *i == "script:u/test-user/active_hist") + .count(), + 0, + "an active path's archived predecessor must not leak into the archived view, got {items:?}" + ); + assert_eq!( + items + .iter() + .filter(|i| *i == "script:u/test-user/archived_path") + .count(), + 1, + "a fully-archived path must appear exactly once (its latest row), got {items:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_archived_favorite_is_pinned_first( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Three fully-archived paths whose names sort aaa < mmm < zzz. Favorite the + // last-sorting one; the archived view pins starred first (each path is one row + // now), so it must lead under name-ascending order despite its late sort key. + for (h, name) in [ + (911111111i64, "aaa"), + (922222222, "mmm"), + (933333333, "zzz"), + ] { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, created_at, language, archived) + VALUES ('test-workspace', $1, $2, $3, '', 'x', 'test-user', '2026-07-19 10:00:00'::timestamptz, 'deno', true)", + ) + .bind(h).bind(format!("u/test-user/{name}")).bind(name) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO favorite (usr, workspace_id, path, favorite_kind) VALUES ('test-user', 'test-workspace', 'u/test-user/zzz', 'script')", + ) + .execute(&db) + .await?; + + let items = paginate_all(port, "order_by=name&order_desc=false&show_archived=true").await; + let pos = |p: &str| items.iter().position(|i| i == p); + let zzz = pos("script:u/test-user/zzz").expect("favorited archived path present"); + let aaa = pos("script:u/test-user/aaa").expect("aaa present"); + let mmm = pos("script:u/test-user/mmm").expect("mmm present"); + assert!( + zzz < aaa && zzz < mmm, + "favorited archived path must be pinned ahead of earlier-named non-favorites, got {items:?}" + ); + Ok(()) +} + +fn new_app(path: &str, summary: &str) -> serde_json::Value { + json!({ + "path": path, + "summary": summary, + "value": {}, + "policy": { "execution_mode": "publisher", "triggerables": {} } + }) +} + +/// A single list request; returns the ordered `type:path` identifiers. +async fn list_once(port: u16, query: &str) -> Vec { + let url = format!("http://localhost:{port}/api/w/test-workspace/runnables/list?{query}"); + let resp = authed(client().get(&url), "SECRET_TOKEN") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "list should succeed for {query}"); + let body: serde_json::Value = resp.json().await.unwrap(); + body["items"] + .as_array() + .unwrap() + .iter() + .map(|it| { + format!( + "{}:{}", + it["type"].as_str().unwrap(), + it["path"].as_str().unwrap() + ) + }) + .collect() +} + +async fn seed_mixed(base: &str, db: &Pool) -> anyhow::Result<()> { + for (p, s) in [ + ("f/alpha/one", "Deploy tool"), + ("f/alpha/two", "Cleanup job"), + ("f/beta/one", "Beta thing"), + ("u/test-user/solo", "Solo script"), + ] { + let r = authed( + client().post(format!("{base}/scripts/create")), + "SECRET_TOKEN", + ) + .json(&new_script(p, s)) + .send() + .await?; + assert_eq!(r.status(), 201, "create script {p}: {}", r.text().await?); + } + let r = authed( + client().post(format!("{base}/flows/create")), + "SECRET_TOKEN", + ) + .json(&new_flow("f/alpha/flowy", "Alpha flow")) + .send() + .await?; + assert_eq!(r.status(), 201, "create flow: {}", r.text().await?); + let r = authed(client().post(format!("{base}/apps/create")), "SECRET_TOKEN") + .json(&new_app("f/beta/appy", "Beta app")) + .send() + .await?; + assert_eq!(r.status(), 201, "create app: {}", r.text().await?); + // Silence unused warnings on db in case future assertions drop it. + let _ = db; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_path_start_scopes_to_folder(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + seed_mixed(&base, &db).await?; + + // path_start scopes to exactly one folder subtree (the folder-navigation path). + let mut alpha = list_once(port, "path_start=f/alpha/").await; + alpha.sort(); + assert_eq!( + alpha, + vec![ + "flow:f/alpha/flowy".to_string(), + "script:f/alpha/one".to_string(), + "script:f/alpha/two".to_string(), + ], + "path_start=f/alpha/ must return only that folder's items" + ); + + // A prefix that matches nothing returns an empty list, not an error. + let empty = list_once(port, "path_start=f/nope/").await; + assert!( + empty.is_empty(), + "unknown folder must be empty, got {empty:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_search_and_kind_filters(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + seed_mixed(&base, &db).await?; + + // Case-insensitive substring search over summary/path. + let deploy = list_once(port, "search=deploy").await; + assert_eq!( + deploy, + vec!["script:f/alpha/one".to_string()], + "search must substring-match the summary only" + ); + + // kinds filter selects a single kind. + let flows = list_once(port, "kinds=flow").await; + assert_eq!(flows, vec!["flow:f/alpha/flowy".to_string()], "kinds=flow"); + let apps = list_once(port, "kinds=app").await; + assert_eq!(apps, vec!["app:f/beta/appy".to_string()], "kinds=app"); + let scripts = list_once(port, "kinds=script").await; + assert_eq!( + scripts.len(), + 4, + "kinds=script -> 4 scripts, got {scripts:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_runnables_starred_pinned_first(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace"); + seed_mixed(&base, &db).await?; + + // Favorite a path that would otherwise sort last (name Z-ish); it must lead. + sqlx::query( + "INSERT INTO favorite (usr, workspace_id, path, favorite_kind) VALUES ('test-user','test-workspace','u/test-user/solo','script')", + ) + .execute(&db) + .await?; + + let items = list_once(port, "order_by=name&order_desc=false").await; + assert_eq!( + items.first().map(String::as_str), + Some("script:u/test-user/solo"), + "starred item pins to the top of the browse view regardless of order, got {items:?}" + ); + Ok(()) +} diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index 704fa0d256..29fe38d2c3 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -352,6 +352,208 @@ mod suspend_resume { Ok(()) } + /// The UI "Resume" button (POST /jobs_u/flow/resume_suspended/:job_id) must reject the + /// triggerer approving their own self_approval_disabled step even when they own the flow + /// path: only admins are exempt from the self-approval restriction, so owning the runnable + /// does not grant the right to self-approve. + #[cfg(feature = "enterprise")] + #[cfg(feature = "deno_core")] + #[sqlx::test(fixtures("base"))] + async fn test_self_approval_disabled_blocks_ui_resume_for_owner( + db: Pool, + ) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_with_self_approval_disabled: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step1'; }" + }, + "suspend": { + "required_events": 1, + "user_auth_required": true, + "self_approval_disabled": true + } + }, { + "id": "b", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step2 - after approval'; }" + } + }] + })) + .unwrap(); + + // Push as a non-admin who owns the flow path, so the owner shortcut is exercised. + let flow = RunJob::from(JobPayload::RawFlow { + value: flow_with_self_approval_disabled, + path: Some("u/test-user-2/test_ui_resume".to_string()), + restarted_from: None, + }) + .push_as(&db, "test-user-2", "test2@windmill.dev") + .await; + + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker( + &db, + async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + + let token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "test-token", + 100, + "test2@windmill.dev", + &Uuid::nil(), + None, + None, + ) + .await + .unwrap(); + + // Resume via the UI endpoint as the owner who triggered the flow. + let response = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}" + )) + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body("{}") + .send() + .await + .unwrap(); + + let status = response.status(); + assert!( + status == reqwest::StatusCode::FORBIDDEN, + "Self-approval via the UI resume endpoint should be blocked for the owner when \ + self_approval_disabled=true. Expected 403 Forbidden, got {}. Response: {}", + status, + response.text().await.unwrap_or_default() + ); + }, + port, + ) + .await; + + server.close().await.unwrap(); + Ok(()) + } + + /// self_approval_disabled must hold even when user_auth_required is not set: the worker must + /// persist the condition and the resume boundary must enforce it for the authenticated + /// triggerer. The triggerer here is a non-owner (folder path they don't own), so the owner + /// shortcut is not involved and this exercises the persistence + authenticated-check path. + #[cfg(feature = "enterprise")] + #[cfg(feature = "deno_core")] + #[sqlx::test(fixtures("base"))] + async fn test_self_approval_disabled_without_user_auth_required( + db: Pool, + ) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // self_approval_disabled without user_auth_required (as a raw-flow/CLI author could set). + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [{ + "id": "a", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step1'; }" + }, + "suspend": { + "required_events": 1, + "self_approval_disabled": true + } + }, { + "id": "b", + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return 'step2 - after approval'; }" + } + }] + })) + .unwrap(); + + // Folder path test-user-2 does not own -> non-owner triggerer (no folders in the base + // fixture), so the owner shortcut is bypassed and only persistence matters here. + let flow = RunJob::from(JobPayload::RawFlow { + value: flow_value, + path: Some("f/system/test_persist".to_string()), + restarted_from: None, + }) + .push_as(&db, "test-user-2", "test2@windmill.dev") + .await; + + let queue = listen_for_queue(&db).await; + let db_ = db.clone(); + + in_test_worker( + &db, + async move { + let db = db_; + + wait_until_flow_suspends(flow, queue, &db).await; + + let token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + "u/test-user-2", + "test-token", + 100, + "test2@windmill.dev", + &Uuid::nil(), + None, + None, + ) + .await + .unwrap(); + + let response = reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs_u/flow/resume_suspended/{flow}" + )) + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body("{}") + .send() + .await + .unwrap(); + + let status = response.status(); + assert!( + status == reqwest::StatusCode::FORBIDDEN, + "Self-approval should be blocked when self_approval_disabled=true even without \ + user_auth_required. Expected 403 Forbidden, got {}. Response: {}", + status, + response.text().await.unwrap_or_default() + ); + }, + port, + ) + .await; + + server.close().await.unwrap(); + Ok(()) + } + /// Test that self-approval WORKS when self_approval_disabled is false (default behavior). /// /// This is the complementary test to test_self_approval_disabled_blocks_owner_resume. diff --git a/backend/tests/wac_approval_resume.rs b/backend/tests/wac_approval_resume.rs new file mode 100644 index 0000000000..436c3d0cf9 --- /dev/null +++ b/backend/tests/wac_approval_resume.rs @@ -0,0 +1,129 @@ +//! WIN-2241: each sequential `wait_for_approval()` in a WAC workflow must +//! resolve to its own approval, not the workflow's first. Guards approved, +//! cancelled and timed-out steps sharing one parent job. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::wac::{WacCheckpoint, WacPendingSteps}; +use windmill_worker::wac_executor::prepare_checkpoint_for_resume; + +async fn insert_resume_row( + db: &Pool, + job_id: Uuid, + resume_id: i32, + approver: &str, + approved: bool, + value: Value, +) -> anyhow::Result<()> { + // resume_id is deliberately arbitrary (a stand-in for the random ids the + // interactive approval channels store) — the fix must not depend on it. + sqlx::query( + "INSERT INTO resume_job (id, job, flow, value, approver, resume_id, approved) \ + VALUES ($1, $2, $2, $3, $4, $5, $6)", + ) + .bind(Uuid::new_v4()) + .bind(job_id) + .bind(value) + .bind(approver) + .bind(resume_id) + .bind(approved) + .execute(db) + .await?; + Ok(()) +} + +fn pending_approval(mut prior: WacCheckpoint, key: &str) -> WacCheckpoint { + prior.pending_steps = Some(WacPendingSteps { + mode: "approval".to_string(), + keys: vec![key.to_string()], + job_ids: Default::default(), + }); + prior +} + +#[sqlx::test] +async fn wac_sequential_approvals_read_own_row(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + // FK target for resume_job.flow and v2_job_status.id (written by save_checkpoint). + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for) VALUES ($1, 'test-workspace', now())", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Step A: approved. + insert_resume_row(&db, job_id, 1, "alice", true, json!({"n": 1})).await?; + let ckpt = pending_approval(WacCheckpoint::default(), "apprA"); + let ckpt = prepare_checkpoint_for_resume(&db, &job_id, ckpt).await?; + assert_eq!(ckpt.completed_steps["apprA"]["approved"], json!(true)); + assert_eq!(ckpt.completed_steps["apprA"]["approver"], json!("alice")); + assert_eq!(ckpt.completed_steps["apprA"]["value"], json!({"n": 1})); + + // Step B: cancelled. Carrying `ckpt` forward preserves consumed_resume_row_ids, + // so B must resolve to its own row, not A's stale approved=true. + insert_resume_row(&db, job_id, 2, "bob", false, json!({"n": 2})).await?; + let ckpt = pending_approval(ckpt, "apprB"); + let ckpt = prepare_checkpoint_for_resume(&db, &job_id, ckpt).await?; + assert_eq!( + ckpt.completed_steps["apprB"]["approved"], + json!(false), + "2nd approval must read its own (cancelled) row, not the 1st's" + ); + assert_eq!(ckpt.completed_steps["apprB"]["approver"], json!("bob")); + + // Step C: timed out — no resume_job row. Falls to the approved=false default. + let ckpt = pending_approval(ckpt, "apprC"); + let ckpt = prepare_checkpoint_for_resume(&db, &job_id, ckpt).await?; + assert_eq!(ckpt.completed_steps["apprC"]["approved"], json!(false)); + assert_eq!(ckpt.completed_steps["apprC"]["approver"], json!(null)); + + Ok(()) +} + +/// A row carrying another step's bound resume_id belongs to that step. The API +/// refuses such resumes but cannot do so atomically with the insert, so +/// consumption must skip them however they got in — while leaving the random ids +/// the interactive channels sign fully eligible. +#[sqlx::test] +async fn wac_approval_skips_another_steps_bound_row(db: Pool) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for) VALUES ($1, 'test-workspace', now())", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, $2)", + ) + .bind(job_id) + .bind(sqlx::types::Json(json!({ + "_minted_approval_keys": { "legal": true, "finance": true } + }))) + .execute(&db) + .await?; + + // Oldest row is bound to `finance`; the later one is a plain Slack-style resume. + insert_resume_row( + &db, + job_id, + windmill_common::wac::approval_resume_id("finance") as i32, + "bob", + true, + json!({"n": "finance"}), + ) + .await?; + insert_resume_row(&db, job_id, 4242, "alice", true, json!({"n": "slack"})).await?; + + let ckpt = pending_approval(WacCheckpoint::default(), "legal"); + let ckpt = prepare_checkpoint_for_resume(&db, &job_id, ckpt).await?; + assert_eq!( + ckpt.completed_steps["legal"]["approver"], + json!("alice"), + "`legal` must skip finance's bound row and take the unbound one" + ); + + Ok(()) +} diff --git a/backend/tests/wac_approval_urls.rs b/backend/tests/wac_approval_urls.rs new file mode 100644 index 0000000000..0cb33e67c0 --- /dev/null +++ b/backend/tests/wac_approval_urls.rs @@ -0,0 +1,267 @@ +//! `jobs/wac_approval_urls/{job}/{step_key}` mints the resume URLs a WAC +//! workflow routes through its own channel. They must address the same +//! `resume_job` record the step's built-in buttons use — i.e. carry +//! `approval_resume_id(step_key)` — and be signed so the unauthenticated resume +//! route accepts them, without that route becoming any easier to forge. + +use sqlx::{Pool, Postgres}; +use windmill_common::wac::approval_resume_id; +use windmill_test_utils::*; + +const WAC_JOB: &str = "a1a1a1a1-a1a1-a1a1-a1a1-a1a1a1a1a1a1"; +// Distinct keys whose SHA-256 prefixes collide in the u32 the resume routes take. +const APPROVAL_COLLISION_A: &str = "approval-12509"; +const APPROVAL_COLLISION_B: &str = "approval-81661"; + +/// The minted URLs point at `BASE_URL`, not the ephemeral test server. +fn to_test_url(base: &str, url: &str) -> String { + let (_, path) = url.split_once("/api/").expect("url has an /api/ segment"); + format!("{base}/{path}") +} + +/// Park the job on `step_key`, as the worker does when that approval suspends. +async fn awaiting_approval(db: &Pool, step_key: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1::uuid, $2) + ON CONFLICT (id) DO UPDATE SET workflow_as_code_status = + v2_job_status.workflow_as_code_status || EXCLUDED.workflow_as_code_status", + ) + .bind(WAC_JOB) + .bind(sqlx::types::Json(serde_json::json!({ + "_checkpoint": { "pending_steps": { "mode": "approval", "keys": [step_key], "job_ids": {} } } + }))) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base", "wac_approval_urls"))] +async fn wac_approval_urls_bind_to_step_key(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api", server.addr.port()); + let client = reqwest::Client::new(); + + let urls: serde_json::Value = client + .get(format!( + "{base}/w/test-workspace/jobs/wac_approval_urls/{WAC_JOB}/manager" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .error_for_status()? + .json() + .await?; + + let resume = urls["resume"].as_str().expect("resume url").to_string(); + let manager_id = approval_resume_id("manager"); + assert!( + resume.contains(&format!("/jobs_u/resume/{WAC_JOB}/{manager_id}/")), + "resume url must carry the step key's resume_id: {resume}" + ); + assert!( + urls["cancel"] + .as_str() + .is_some_and(|c| c.contains(&format!("/jobs_u/cancel/{WAC_JOB}/{manager_id}/"))), + "cancel url must carry the same resume_id: {urls}" + ); + assert_ne!( + manager_id, + approval_resume_id("finance"), + "distinct approval steps must not share a resume_job record" + ); + + // A signature is only valid for the resume_id it was minted for, so the URL + // can't be retargeted at another step of the same workflow. + let retargeted = resume.replace( + &format!("/{manager_id}/"), + &format!("/{}/", approval_resume_id("finance")), + ); + let status = client + .post(to_test_url(&base, &retargeted)) + .json(&serde_json::json!({})) + .send() + .await? + .status(); + assert!( + !status.is_success(), + "signature minted for `manager` must not resume another step (got {status})" + ); + + // The genuine URL resumes without any credential — possession of the + // signature is the authority, as for the built-in approval buttons. + awaiting_approval(&db, "manager").await?; + let resp = client + .post(to_test_url(&base, &resume)) + .json(&serde_json::json!({ "ok": true })) + .send() + .await?; + assert!( + resp.status().is_success(), + "minted resume url must be accepted: {} {}", + resp.status(), + resp.text().await.unwrap_or_default() + ); + + let (approved, value): (bool, sqlx::types::Json) = sqlx::query_as( + "SELECT approved, value FROM resume_job WHERE job = $1::uuid AND resume_id = $2", + ) + .bind(WAC_JOB) + .bind(manager_id as i32) + .fetch_one(&db) + .await?; + assert!(approved); + assert_eq!(value.0, serde_json::json!({ "ok": true })); + + Ok(()) +} + +/// Approval rows are consumed oldest-first regardless of resume_id (WIN-2241), so +/// a URL minted for a later step and clicked while an earlier one is pending would +/// otherwise resolve the earlier step with this approver's answer. +#[sqlx::test(fixtures("base", "wac_approval_urls"))] +async fn wac_approval_url_for_another_step_is_rejected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api", server.addr.port()); + let client = reqwest::Client::new(); + + // The workflow mints both approvals' URLs up front, then suspends on `legal`. + let mut minted = Vec::new(); + for step_key in ["legal", "finance"] { + let urls: serde_json::Value = client + .get(format!( + "{base}/w/test-workspace/jobs/wac_approval_urls/{WAC_JOB}/{step_key}" + )) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await? + .error_for_status()? + .json() + .await?; + minted.push(urls["resume"].as_str().expect("resume url").to_string()); + } + // Nothing pending yet: the link must not bank a row that the next approval + // to be reached would consume, whichever step that turns out to be. + let resp = client + .post(to_test_url(&base, &minted[1])) + .json(&serde_json::json!({})) + .send() + .await?; + assert_eq!( + resp.status(), + reqwest::StatusCode::BAD_REQUEST, + "a bound link must not be bankable before its step awaits approval" + ); + + awaiting_approval(&db, "legal").await?; + + let resp = client + .post(to_test_url(&base, &minted[1])) + .json(&serde_json::json!({})) + .send() + .await?; + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "finance's url must not resume the pending `legal` step: {body}" + ); + assert!( + body.contains("finance"), + "error must name the bound step: {body}" + ); + + // The pending step's own url still works. + let resp = client + .post(to_test_url(&base, &minted[0])) + .json(&serde_json::json!({})) + .send() + .await?; + assert!( + resp.status().is_success(), + "the pending step's url must still resume: {}", + resp.text().await.unwrap_or_default() + ); + + Ok(()) +} + +/// The guards around minting: a step key must be non-empty, the job must be in the +/// caller's workspace and be WAC-shaped, and two keys may not share a resume id. +#[sqlx::test(fixtures("base", "wac_approval_urls"))] +async fn wac_approval_urls_mint_guards(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api", server.addr.port()); + let client = reqwest::Client::new(); + let mint = |ws: &str, job: &str, key: &str| { + let url = format!("{base}/w/{ws}/jobs/wac_approval_urls/{job}/{key}"); + client.get(url).header("Authorization", "Bearer SECRET_TOKEN").send() + }; + + assert_eq!( + mint("test-workspace", WAC_JOB, "%20").await?.status(), + reqwest::StatusCode::BAD_REQUEST, + "a blank step key must be refused rather than silently meaning `approval`" + ); + + // `v2_job_status` is keyed by job id alone, so the mint must not accept a job + // from another workspace and stamp its status row. + assert_eq!( + mint("test-workspace-2", WAC_JOB, "manager").await?.status(), + reqwest::StatusCode::NOT_FOUND, + "a job outside the caller's workspace must not be mintable" + ); + + // APPROVAL_COLLISION_A and _B hash to the same 32-bit resume id, so they would + // share one resume_job row and one capability. + assert!(mint("test-workspace", WAC_JOB, APPROVAL_COLLISION_A) + .await? + .status() + .is_success()); + let resp = mint("test-workspace", WAC_JOB, APPROVAL_COLLISION_B).await?; + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + assert_eq!(status, reqwest::StatusCode::BAD_REQUEST, "colliding key: {body}"); + assert!(body.contains(APPROVAL_COLLISION_A), "error must name the other key: {body}"); + + Ok(()) +} + +/// Colliding keys share one resume_job row and one capability, so the mint that +/// records them must let exactly one through however the requests interleave. +#[sqlx::test(fixtures("base", "wac_approval_urls"))] +async fn wac_concurrent_colliding_mints_admit_exactly_one(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base = format!("http://localhost:{}/api", server.addr.port()); + + let mut set = tokio::task::JoinSet::new(); + for key in [APPROVAL_COLLISION_A, APPROVAL_COLLISION_B] { + let url = format!("{base}/w/test-workspace/jobs/wac_approval_urls/{WAC_JOB}/{key}"); + set.spawn(async move { + reqwest::Client::new() + .get(url) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await + .map(|r| r.status()) + }); + } + let mut ok = 0; + let mut rejected = 0; + while let Some(res) = set.join_next().await { + match res?? { + s if s.is_success() => ok += 1, + reqwest::StatusCode::BAD_REQUEST => rejected += 1, + other => anyhow::bail!("unexpected status {other}"), + } + } + assert_eq!((ok, rejected), (1, 1), "exactly one colliding key may be minted"); + Ok(()) +} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 2349d476b7..3140cd2d7f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5338,9 +5338,10 @@ async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { /// This validates that `check_tag_available_for_workspace_internal` is properly called /// when pushing jobs from worker_flow. #[sqlx::test(fixtures("base"))] +#[serial] async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow::Result<()> { use windmill_common::worker::{ - CustomTags, SpecificTagData, SpecificTagType, CUSTOM_TAGS_PER_WORKSPACE, + CustomTags, SpecificTagData, SpecificTagType, WorkspaceMatcher, CUSTOM_TAGS_PER_WORKSPACE, }; initialize_tracing().await; @@ -5354,7 +5355,10 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: "restricted-tag".to_string(), SpecificTagData { tag_type: SpecificTagType::NoneExcept, - workspaces: vec!["other-workspace".to_string()], + workspaces: vec![WorkspaceMatcher { + id: "other-workspace".to_string(), + include_forks: false, + }], }, )]), })); @@ -5404,6 +5408,61 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } +/// The `*` fork marker only grants through a real `parent_workspace_id` lineage lookup, which the +/// parse-level unit tests cannot reach: they hand `applies_to_workspace` a synthetic chain, so a +/// regression in the lookup or in the `is_fork_scoped()` gate that skips it would pass them. +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> anyhow::Result<()> { + use windmill_common::jobs::check_tag_available_for_workspace_internal; + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. + let fork = "wm-fork-tagmarker"; + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ($1, $1, 'test-user', 'test-workspace')", + fork + ) + .execute(&db) + .await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "forky(test-workspace*)".to_string(), + "bare(test-workspace)".to_string(), + ]))); + + // test2 is not a superadmin, who would bypass the scope check entirely. + let email = "test2@windmill.dev"; + + for (w_id, tag) in [("test-workspace", "bare"), ("test-workspace", "forky")] { + assert!( + check_tag_available_for_workspace_internal(&db, w_id, tag, email, None) + .await + .is_ok(), + "{tag} should be available in the workspace it names" + ); + } + assert!( + check_tag_available_for_workspace_internal(&db, fork, "forky", email, None) + .await + .is_ok(), + "a `*` tag must be granted to a fork through its parent lineage" + ); + assert!( + check_tag_available_for_workspace_internal(&db, fork, "bare", email, None) + .await + .is_err(), + "an unmarked tag must not reach a fork of the workspace it names" + ); + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_whileloop_propagates_inner_iterator_eval_failure( diff --git a/backend/tests/zombie_flow_recovery.rs b/backend/tests/zombie_flow_recovery.rs new file mode 100644 index 0000000000..fb658dfb03 --- /dev/null +++ b/backend/tests/zombie_flow_recovery.rs @@ -0,0 +1,661 @@ +//! Regression test for hand-recovery of between-steps zombie flows. +//! +//! When a worker is OOM-killed mid state-transition, the zombie monitor +//! (`handle_zombie_flows` → `cancel_job` with force) reaps the flow: it lands in +//! `v2_job_completed` as `canceled`, with its `flow_status` preserved: the step +//! whose transition was lost stays `InProgress` even though all its children +//! completed successfully. This test reproduces that exact terminal state and +//! asserts that a hand-restart from the stuck step reuses every completed child +//! (no re-run) and the flow reaches success. +//! +//! The reaper itself lives in the `windmill` binary crate and is unreachable +//! from an integration test, so we reproduce the state `cancel_job(force)` +//! leaves behind directly; the fix under test is the restart-resolution path, +//! not the detection query. + +#![cfg(feature = "deno_core")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::flow_status::{BranchChosen, FlowStatus, RestartedFrom}; +use windmill_common::flows::FlowValue; +use windmill_common::jobs::JobPayload; +use windmill_test_utils::*; + +/// Child job UUID for a top-level step in a completed flow's `flow_status` +/// (optionally the iteration index for a ForLoop / BranchAll container). +async fn child_job_id_for_step( + db: &Pool, + flow_job_id: uuid::Uuid, + step_id: &str, + iter: Option, +) -> uuid::Uuid { + let raw: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + flow_job_id + ) + .fetch_one(db) + .await + .unwrap() + .expect("flow_status missing"); + let status: FlowStatus = serde_json::from_value(raw).expect("parse flow_status"); + let module = status + .modules + .iter() + .find(|m| m.id() == step_id) + .expect("step in flow_status"); + match iter { + Some(i) => module.flow_jobs().expect("flow_jobs")[i], + None => module.job().expect("job"), + } +} + +/// A between-steps zombie whose fan-out completed but whose final transition was +/// lost can be hand-restarted from the stuck step, reusing every completed child +/// (including the last iteration) and reaching success. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_between_steps_zombie_restart_reuses_all_children( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // A fan-out ForLoop `fanout` (2 iterations) followed by `after`, which + // consumes the loop's aggregated result. In the zombie scenario `fanout` + // finished all iterations but its final transition was lost, so `after` + // never ran. + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + // Run to completion to obtain real, successful child jobs. + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success, "baseline run should succeed"); + assert_eq!(full_run.json_result().unwrap(), json!("a,b")); + + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + let orig_iter1 = child_job_id_for_step(&db, full_run.id, "fanout", Some(1)).await; + let orig_after = child_job_id_for_step(&db, full_run.id, "after", None).await; + + // Reproduce the zombie-reaper's terminal state: cancelled by `monitor` with + // `flow_status` frozen mid-transition: `fanout` still `InProgress` (all + // iterations done), `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + // A reaped loop keeps its cursor at the last iteration. + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed + SET status = 'canceled', canceled_by = 'monitor', canceled_reason = 'zombie flow', + flow_status = $2 + WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + // Hand-restart from the stuck step. `fanout` is recognised as a derivable + // between-steps zombie (all children succeeded), so it is reused verbatim and + // only the dropped transition onward is replayed. `Some(0)` is the exact value the + // run page's "Re-start from" button sends (a whole-step restart), not `None`. + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: Some(0), + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + // Flow reaches success, reusing the loop's aggregated result. + assert!( + restarted.success, + "restarted zombie flow should succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("a,b")); + + // Every completed loop iteration reuses its original child job (no re-run); + // only `after`, which never ran, executes fresh. + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + let new_iter1 = child_job_id_for_step(&db, restarted.id, "fanout", Some(1)).await; + let new_after = child_job_id_for_step(&db, restarted.id, "after", None).await; + assert_eq!(new_iter0, orig_iter0, "loop iteration 0 must be reused"); + assert_eq!(new_iter1, orig_iter1, "loop iteration 1 must be reused"); + assert_ne!(new_after, orig_after, "`after` should run fresh"); + + Ok(()) +} + +/// A serial for-loop reaped *between* iterations (an all-success prefix, but the +/// cursor not yet at the last iteration) must NOT be treated as complete: reuse +/// would silently drop the remaining iterations. A downstream `after` step makes +/// the loop non-final, so the ONLY thing that can prevent reuse here is the +/// cursor-completeness guard; if it regresses, `after` would consume a truncated +/// loop result and this test fails. Restart must re-run the whole loop instead. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b', 'c']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", + "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + // Reap after iteration 0: the loop is InProgress with the cursor still on + // iteration 0 (of 3), only iteration 0 recorded; `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 0, "itered_len": 3 }); + m["flow_jobs"] = json!([m["flow_jobs"][0]]); + m["flow_jobs_success"] = json!([true]); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + // The loop re-runs from scratch: all three iterations execute (so `after` sees + // "a,b,c", not a truncated "a"), and iteration 0 is a fresh job. + assert!( + restarted.success, + "restart should re-run the loop and succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("a,b,c")); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "iteration 0 must re-run, not be reused" + ); + + Ok(()) +} + +/// A nested restart request targets an inner step of the restart-step container. +/// Even when that container is an eligible between-steps zombie, reuse must NOT +/// fire (it would skip the whole container and ignore the explicit nested target). +/// The inner step must re-run. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_nested_restart_not_swallowed_by_zombie_reuse( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // `branch` is a BranchOne (single child, so branch_or_iteration_n is None on + // restart: the exact shape that would trip zombie reuse) with two inner steps, + // followed by a downstream `after`. + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "branch", + "value": { + "type": "branchone", + "default": [], + "branches": [{ + "expr": "true", + "modules": [ + { + "id": "inner_first", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": {}, + "content": "export function main() { return 'first' }" + } + }, + { + "id": "inner_second", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "first": { "type": "javascript", "expr": "results.inner_first" } + }, + "content": "export function main(first: string) { return `${first}|second` }" + } + } + ] + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "b": { "type": "javascript", "expr": "results.branch" } + }, + "content": "export function main(b: string) { return `after:${b}` }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + assert_eq!(full_run.json_result().unwrap(), json!("after:first|second")); + let branch_child = child_job_id_for_step(&db, full_run.id, "branch", None).await; + let orig_inner_second = child_job_id_for_step(&db, branch_child, "inner_second", None).await; + + // Reap `branch` as a between-steps zombie (its child completed, transition lost); + // `after` never reached. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("branch") => m["type"] = json!("InProgress"), + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + // Nested restart: re-run `inner_second` inside `branch`. Zombie reuse must step + // aside so the nested chain is honored. + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "branch".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: Some(BranchChosen::Branch { branch: 0 }), + nested: Some(Box::new(RestartedFrom { + flow_job_id: branch_child, + step_id: "inner_second".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + })), + }) + .run_until_complete(&db, false, port) + .await; + + assert!( + restarted.success, + "nested restart of a zombie container should succeed: {:?}", + restarted.json_result() + ); + assert_eq!( + restarted.json_result().unwrap(), + json!("after:first|second") + ); + let new_branch_child = child_job_id_for_step(&db, restarted.id, "branch", None).await; + let new_inner_second = child_job_id_for_step(&db, new_branch_child, "inner_second", None).await; + assert_ne!( + new_inner_second, orig_inner_second, + "the nested target inner_second must re-run, not be skipped by zombie reuse" + ); + + Ok(()) +} + +/// A raw-flow (editor preview) restart queues the request's CURRENT definition, which the editor +/// allows to differ from the completed run. Zombie reuse must not fire there: it would validate the +/// stored step and synthesize Success from the old children, skipping the user's edit. The edited +/// step must re-run and downstream must observe its new result. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_raw_flow_restart_does_not_reuse_edited_step( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_of = |suffix: &str| -> FlowValue { + serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": format!("export function main(v: string) {{ return v + '{suffix}' }}") + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap() + }; + + let full_run = + RunJob::from(JobPayload::RawFlow { value: flow_of(""), path: None, restarted_from: None }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + assert_eq!(full_run.json_result().unwrap(), json!("a,b")); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'monitor', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RawFlow { + value: flow_of("X"), + path: None, + restarted_from: Some(RestartedFrom { + flow_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }), + }) + .run_until_complete(&db, false, port) + .await; + + // The edited step must run: results reflect the new definition, not the reused old children. + assert!( + restarted.success, + "edited raw-flow restart should succeed: {:?}", + restarted.json_result() + ); + assert_eq!(restarted.json_result().unwrap(), json!("aX,bX")); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "the edited fanout step must re-run, not be reused" + ); + + Ok(()) +} + +/// Only a flow reaped by the zombie monitor (canceled_by = 'monitor') is eligible for reuse. A +/// plain force-cancel at the same boundary (a child succeeded, its parent transition not yet +/// landed) yields the identical InProgress/all-success shape but must keep restart-from-step +/// semantics: the selected step re-runs. +#[sqlx::test(fixtures("base", "hello"))] +async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let flow_value: FlowValue = serde_json::from_value(json!({ + "modules": [ + { + "id": "fanout", + "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "['a', 'b']" }, + "skip_failures": false, + "parallel": false, + "modules": [{ + "id": "inner", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "v": { "type": "javascript", "expr": "flow_input.iter.value" } + }, + "content": "export function main(v: string) { return v }" + } + }] + } + }, + { + "id": "after", + "value": { + "type": "rawscript", "language": "deno", + "input_transforms": { + "loop_res": { "type": "javascript", "expr": "results.fanout" } + }, + "content": "export function main(loop_res: string[]) { return loop_res.join(',') }" + } + } + ] + })) + .unwrap(); + + let full_run = RunJob::from(JobPayload::RawFlow { + value: flow_value.clone(), + path: None, + restarted_from: None, + }) + .run_until_complete(&db, false, port) + .await; + assert!(full_run.success); + let orig_iter0 = child_job_id_for_step(&db, full_run.id, "fanout", Some(0)).await; + + // Same frozen-transition shape as a zombie, but canceled by a USER, not the monitor. + let mut flow_status: serde_json::Value = sqlx::query_scalar!( + "SELECT flow_status FROM v2_job_completed WHERE id = $1", + full_run.id + ) + .fetch_one(&db) + .await? + .expect("flow_status"); + for m in flow_status["modules"].as_array_mut().unwrap() { + match m["id"].as_str() { + Some("fanout") => { + m["type"] = json!("InProgress"); + m["iterator"] = json!({ "index": 1, "itered_len": 2 }); + } + Some("after") => *m = json!({ "type": "WaitingForPriorSteps", "id": "after" }), + _ => {} + } + } + flow_status["step"] = json!(0); + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled', canceled_by = 'admin', + flow_status = $2 WHERE id = $1", + full_run.id, + flow_status, + ) + .execute(&db) + .await?; + + let restarted = RunJob::from(JobPayload::RestartedFlow { + completed_job_id: full_run.id, + step_id: "fanout".into(), + branch_or_iteration_n: None, + flow_version: None, + branch_chosen: None, + nested: None, + }) + .run_until_complete(&db, false, port) + .await; + + assert!(restarted.success); + let new_iter0 = child_job_id_for_step(&db, restarted.id, "fanout", Some(0)).await; + assert_ne!( + new_iter0, orig_iter0, + "a non-monitor cancel must re-run the step, not reuse the child" + ); + + Ok(()) +} diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index d71de2f35f..a255f47a6d 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -6,7 +6,10 @@ use crate::{ query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, should_use_structured_output_tool, AI_HTTP_HEADERS}, + utils::{ + collect_system_prompt, extract_text_content, should_use_structured_output_tool, + AI_HTTP_HEADERS, + }, }; use async_trait::async_trait; use http::Method; @@ -209,7 +212,7 @@ fn convert_messages_to_anthropic(messages: &[OpenAIMessage]) -> Vec { - // Skip - handled via args.system_prompt in build_text_request + // Lifted into the request's top-level `system` field by build_text_request } "user" => { // Convert user messages @@ -601,19 +604,17 @@ impl AnthropicQueryBuilder { } } - // Build system content from system_prompt, but None if system_prompt is empty string - let system = match args.system_prompt { - Some(s) if !s.is_empty() => Some(vec![AnthropicSystemContent { + let system = collect_system_prompt(&prepared_messages, args.system_prompt).map(|text| { + vec![AnthropicSystemContent { r#type: "text".to_string(), - text: s.to_string(), + text, cache_control: if self.is_vertex() { None } else { Some(CacheControl::ephemeral()) }, - }]), - _ => None, - }; + }] + }); // Check if we need to force tool usage for structured output let has_output_properties = args @@ -842,6 +843,88 @@ mod tests { } } + const SYSTEM_PROMPT: &str = "You are a helpful assistant"; + + fn authed_client() -> AuthedClient { + AuthedClient::new( + "http://localhost:8000".to_string(), + "test-workspace".to_string(), + "token".to_string(), + None, + ) + } + + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + let args = BuildRequestArgs { + messages, + tools: None, + model: "claude-sonnet-4", + temperature: None, + reasoning_effort: None, + max_tokens: None, + output_schema: None, + output_type: &OutputType::Text, + system_prompt, + user_message: "hello", + attachments: None, + has_websearch: false, + }; + + AnthropicQueryBuilder::new(AIProvider::Anthropic, AIPlatform::Standard) + .build_request(&args, &authed_client(), "test-workspace") + .await + .unwrap() + } + + /// The worker prepends the system prompt as a system message *and* passes it as + /// `system_prompt`; the request must still carry it exactly once. + #[tokio::test] + async fn sends_system_prompt_only_in_system_field() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["text"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let sent = request["messages"].as_array().unwrap(); + assert!(sent.iter().all(|message| message["role"] != "system")); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0]["role"], "user"); + } + + /// Manual-memory conversations supply their own system messages without a + /// `system_prompt` arg: those must still reach the model. + #[tokio::test] + async fn lifts_manual_system_messages_into_system_field() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["system"][0]["text"], "be terse"); + assert_eq!(request["messages"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn omits_system_without_a_system_prompt() { + let messages = vec![message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(request.get("system").is_none()); + } + fn has_header(headers: &[(String, String)], name: &str, value: &str) -> bool { headers .iter() diff --git a/backend/windmill-ai/src/providers/openai.rs b/backend/windmill-ai/src/providers/openai.rs index bdb882679b..8ddd752b70 100644 --- a/backend/windmill-ai/src/providers/openai.rs +++ b/backend/windmill-ai/src/providers/openai.rs @@ -6,7 +6,7 @@ use crate::{ query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, sse::{OpenAIResponsesSSEParser, SSEParser}, types::*, - utils::extract_text_content, + utils::{collect_system_prompt, extract_text_content}, }; use async_trait::async_trait; use serde::{Deserialize, Serialize}; @@ -206,7 +206,7 @@ pub struct ResponsesApiRequest<'a> { pub model: &'a str, pub input: Vec, #[serde(skip_serializing_if = "Option::is_none")] - pub instructions: Option<&'a str>, + pub instructions: Option, pub tools: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub stream: Option, @@ -371,9 +371,19 @@ impl OpenAIQueryBuilder { let prepared_messages = prepare_messages_for_api(args.messages, client, workspace_id).await?; + // Only the system prompt leading the conversation moves to `instructions`; echoing it in + // `input` as well would send it twice. This API accepts system messages anywhere in + // `input`, so any later one stays where the caller put it, position and content intact. + let leading_system = prepared_messages + .iter() + .take_while(|message| message.role == "system") + .count(); + let instructions = + collect_system_prompt(&prepared_messages[..leading_system], args.system_prompt); + // Convert full message history to Responses API input format // (following frontend pattern from openai-responses.ts) - let input_items = convert_messages_to_responses_input(&prepared_messages); + let input_items = convert_messages_to_responses_input(&prepared_messages[leading_system..]); // Build tools array using typed structs let mut tools: Vec = Vec::new(); @@ -416,7 +426,7 @@ impl OpenAIQueryBuilder { let request = ResponsesApiRequest { model: args.model, input: input_items, - instructions: args.system_prompt, // System prompt goes to instructions field + instructions, tools, stream: Some(true), temperature: args.temperature, @@ -474,7 +484,7 @@ impl OpenAIQueryBuilder { let request = ResponsesApiRequest { model: args.model, input: vec![ResponsesApiInputItem::InputMessage { role: "user".to_string(), content }], - instructions: args.system_prompt, + instructions: args.system_prompt.map(str::to_string), tools, stream: None, // Image generation doesn't use streaming temperature: args.temperature, @@ -508,9 +518,7 @@ impl QueryBuilder for OpenAIQueryBuilder { parser.parse_events(response).await?; // Convert OpenAI Responses usage to TokenUsage - let usage = parser - .usage - .map(|u| TokenUsage::new(u.input_tokens, u.output_tokens, u.total_tokens)); + let usage = parser.usage.map(|u| u.to_token_usage()); Ok(ParsedResponse::Text { content: if parser.accumulated_content.is_empty() { @@ -582,3 +590,114 @@ impl QueryBuilder for OpenAIQueryBuilder { vec![("Authorization", format!("Bearer {}", api_key))] } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::query_builder::QueryBuilder; + + const SYSTEM_PROMPT: &str = "You are a helpful assistant"; + + fn client() -> AuthedClient { + AuthedClient::new( + "http://localhost:8000".to_string(), + "test-workspace".to_string(), + "token".to_string(), + None, + ) + } + + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + async fn build_text_body(messages: &[OpenAIMessage], system_prompt: Option<&str>) -> String { + let args = BuildRequestArgs { + messages, + tools: None, + model: "gpt-5", + temperature: None, + reasoning_effort: None, + max_tokens: None, + output_schema: None, + output_type: &OutputType::Text, + system_prompt, + user_message: "hello", + attachments: None, + has_websearch: false, + }; + + OpenAIQueryBuilder::new(AIProvider::OpenAI) + .build_request(&args, &client(), "test-workspace") + .await + .unwrap() + } + + /// The worker prepends the system prompt as a system message *and* passes it as + /// `system_prompt`; the request must still carry it exactly once. + #[tokio::test] + async fn sends_system_prompt_only_in_instructions() { + let messages = vec![message("system", SYSTEM_PROMPT), message("user", "hi")]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let input = request["input"].as_array().unwrap(); + assert!(input.iter().all(|item| item["role"] != "system")); + assert_eq!(input.len(), 1); + assert_eq!(input[0]["role"], "user"); + } + + /// This API takes system messages anywhere in `input`, so a late steering message keeps + /// its position instead of being hoisted into `instructions`. + #[tokio::test] + async fn keeps_a_mid_conversation_system_message_in_place() { + let messages = vec![ + message("system", SYSTEM_PROMPT), + message("user", "hi"), + message("system", "answer in one word"), + message("user", "and now?"), + ]; + + let body = build_text_body(&messages, Some(SYSTEM_PROMPT)).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], SYSTEM_PROMPT); + assert_eq!(body.matches(SYSTEM_PROMPT).count(), 1); + + let input = request["input"].as_array().unwrap(); + assert_eq!(input.len(), 3); + assert_eq!(input[1]["role"], "system"); + assert_eq!(input[1]["content"][0]["text"], "answer in one word"); + } + + /// Manual-memory conversations supply their own system messages without a + /// `system_prompt` arg: those must still reach the model. + #[tokio::test] + async fn lifts_manual_system_messages_into_instructions() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!(request["instructions"], "be terse"); + assert_eq!(request["input"].as_array().unwrap().len(), 1); + } + + #[tokio::test] + async fn omits_instructions_without_a_system_prompt() { + let messages = vec![message("user", "hi")]; + + let body = build_text_body(&messages, None).await; + let request: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert!(request.get("instructions").is_none()); + } +} diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index f641daec64..790bcb6c28 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -269,8 +269,7 @@ impl QueryBuilder for OtherQueryBuilder { } // Convert OpenAI Chat Completions usage to TokenUsage - let usage = openai_usage - .map(|u| TokenUsage::new(u.prompt_tokens, u.completion_tokens, u.total_tokens)); + let usage = openai_usage.map(|u| u.to_token_usage()); Ok(ParsedResponse::Text { content: if accumulated_content.is_empty() { diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index 66ff7da9de..96692d0173 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -13,7 +13,7 @@ use crate::{ AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall, }, query_builder::StreamEventSink, - types::StreamingEvent, + types::{StreamingEvent, TokenUsage}, }; #[derive(Deserialize)] @@ -44,6 +44,14 @@ pub struct OpenAIChoice { pub delta: Option, } +/// Nested prompt token details returned by the Chat Completions API. +/// `cached_tokens` is the portion of `prompt_tokens` served from cache (a subset, not additive). +#[derive(Deserialize, Debug, Clone, Default)] +pub struct OpenAIPromptTokensDetails { + #[serde(default)] + pub cached_tokens: Option, +} + /// OpenAI Chat Completions API usage information (from final chunk with stream_options.include_usage) #[derive(Deserialize, Debug, Clone, Default)] pub struct OpenAIChatUsage { @@ -53,6 +61,24 @@ pub struct OpenAIChatUsage { pub completion_tokens: Option, #[serde(default)] pub total_tokens: Option, + #[serde(default)] + pub prompt_tokens_details: Option, +} + +impl OpenAIChatUsage { + /// cached_tokens is a subset of prompt_tokens, so input/total are reported as-is + /// and only recorded as cache_read for reporting. + pub fn to_token_usage(self) -> TokenUsage { + TokenUsage::new( + self.prompt_tokens, + self.completion_tokens, + self.total_tokens, + ) + .with_cache( + self.prompt_tokens_details.and_then(|d| d.cached_tokens), + None, + ) + } } #[derive(Deserialize)] @@ -684,6 +710,14 @@ pub struct OpenAIUrlCitationEvent { pub title: Option, } +/// Nested input token details returned by the Responses API. +/// `cached_tokens` is the portion of `input_tokens` served from cache (a subset, not additive). +#[derive(Deserialize, Debug, Clone, Default)] +pub struct OpenAIInputTokensDetails { + #[serde(default)] + pub cached_tokens: Option, +} + /// OpenAI Responses API usage information #[derive(Deserialize, Debug, Clone)] pub struct OpenAIResponsesUsage { @@ -693,6 +727,19 @@ pub struct OpenAIResponsesUsage { pub output_tokens: Option, #[serde(default)] pub total_tokens: Option, + #[serde(default)] + pub input_tokens_details: Option, +} + +impl OpenAIResponsesUsage { + /// cached_tokens is a subset of input_tokens, so input/total are reported as-is + /// and only recorded as cache_read for reporting. + pub fn to_token_usage(self) -> TokenUsage { + TokenUsage::new(self.input_tokens, self.output_tokens, self.total_tokens).with_cache( + self.input_tokens_details.and_then(|d| d.cached_tokens), + None, + ) + } } /// OpenAI Responses API response object (from response.completed event) @@ -927,6 +974,34 @@ mod tests { assert_eq!(json["content"], "hmm"); } + #[test] + fn openai_chat_usage_maps_cached_prompt_tokens() { + // Payload shape returned by OpenAI and Azure OpenAI Chat Completions. + // cached_tokens lives under prompt_tokens_details and is a subset of prompt_tokens, + // so it must land in cache_read while input/total stay as the provider reported them. + let usage: OpenAIChatUsage = serde_json::from_str( + r#"{"prompt_tokens":4819,"completion_tokens":1,"total_tokens":4820,"prompt_tokens_details":{"cached_tokens":4736,"audio_tokens":0}}"#, + ) + .unwrap(); + let token_usage = usage.to_token_usage(); + assert_eq!(token_usage.cache_read_input_tokens, Some(4736)); + assert_eq!(token_usage.input_tokens, Some(4819)); + assert_eq!(token_usage.total_tokens, Some(4820)); + } + + #[test] + fn openai_responses_usage_maps_cached_input_tokens() { + // Payload shape returned by the OpenAI Responses API. + let usage: OpenAIResponsesUsage = serde_json::from_str( + r#"{"input_tokens":4819,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":4736},"output_tokens":2,"total_tokens":4821}"#, + ) + .unwrap(); + let token_usage = usage.to_token_usage(); + assert_eq!(token_usage.cache_read_input_tokens, Some(4736)); + assert_eq!(token_usage.input_tokens, Some(4819)); + assert_eq!(token_usage.total_tokens, Some(4821)); + } + #[test] fn openai_delta_parses_reasoning_content() { // DeepSeek and similar stream reasoning under `reasoning_content`. diff --git a/backend/windmill-ai/src/utils.rs b/backend/windmill-ai/src/utils.rs index fab1644255..328d2e0458 100644 --- a/backend/windmill-ai/src/utils.rs +++ b/backend/windmill-ai/src/utils.rs @@ -1,6 +1,6 @@ use crate::{ ai_providers::AIProvider, - ai_types::{ContentPart, OpenAIContent}, + ai_types::{ContentPart, OpenAIContent, OpenAIMessage}, }; use windmill_common::utils::configure_client; @@ -18,20 +18,19 @@ lazy_static::lazy_static! { /// use it instead of the shared `HTTP_CLIENT`. Redirects are governed by /// `ALLOW_AI_BASE_URL_REDIRECTS` (disabled by default). Mirrors the API proxy /// client (windmill-api/src/ai.rs). + /// + /// This pooled client does no DNS pinning: callers reaching a user-controlled + /// base_url must go through [`pinned_ai_client_for`] so the connect targets + /// the SSRF-validated address (DNS-rebinding TOCTOU). It is the safe default + /// only for trusted/fixed hosts. pub static ref AI_HTTP_CLIENT: reqwest::Client = { - let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS { + if *ALLOW_AI_BASE_URL_REDIRECTS { tracing::warn!( "ALLOW_AI_BASE_URL_REDIRECTS is enabled - the AI HTTP client will follow \ redirects, weakening SSRF protection on provider base URLs" ); - reqwest::redirect::Policy::default() - } else { - reqwest::redirect::Policy::none() - }; - configure_client(reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .connect_timeout(std::time::Duration::from_secs(10)) - .redirect(redirect)) + } + ai_http_client_builder() .build() .expect("Failed to build AI HTTP client - check system TLS configuration") }; @@ -64,11 +63,103 @@ lazy_static::lazy_static! { }; } +/// Shared configuration for every client that targets a user-configured AI +/// provider `base_url` (the pooled [`AI_HTTP_CLIENT`] and per-request DNS-pinned +/// clients from [`pinned_ai_client_for`]). Redirects are disabled by default +/// because the SSRF check on base_url is single-shot; DNS pinning likewise only +/// covers the original host, so a redirect could bounce a validated public host +/// into a private/internal one (see `ALLOW_AI_BASE_URL_REDIRECTS`). +pub fn ai_http_client_builder() -> reqwest::ClientBuilder { + let redirect = if *ALLOW_AI_BASE_URL_REDIRECTS { + reqwest::redirect::Policy::default() + } else { + reqwest::redirect::Policy::none() + }; + configure_client( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .connect_timeout(std::time::Duration::from_secs(10)) + .redirect(redirect), + ) +} + +/// Build the client for a single outbound AI request to `url`, pinning DNS to +/// the SSRF-validated address so the connect cannot rebind to an internal IP +/// after the check (DNS-rebinding TOCTOU). +/// +/// Returns the shared pooled [`AI_HTTP_CLIENT`] unchanged when there is nothing +/// to pin — an IP-literal host, or a deployment that opted into private AI +/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint +/// as `AIProvider::get_base_url` apply, so this is consistent with the +/// credential-time validation while additionally closing the connect-time window. +pub async fn pinned_ai_client_for( + url: &str, +) -> windmill_common::error::Result> { + use std::borrow::Cow; + use windmill_common::error::{to_anyhow, Error}; + use windmill_common::ssrf::SsrfValidationError; + + if *crate::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { + return Ok(Cow::Borrowed(&AI_HTTP_CLIENT)); + } + + let target = windmill_common::ssrf::validate_url_for_ssrf(url) + .await + .map_err(|e| match e { + e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( + "{e}. If you need to use private/internal AI endpoints, \ + set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" + )), + e => Error::from(e), + })?; + + if target.pinned_addrs().is_empty() { + return Ok(Cow::Borrowed(&AI_HTTP_CLIENT)); + } + + let client = target + .apply_dns_pinning(ai_http_client_builder()) + .build() + .map_err(to_anyhow)?; + Ok(Cow::Owned(client)) +} + /// AWS Bedrock do not handle structured output query param, so we use a tool for structured output. Same for every Claude models. pub fn should_use_structured_output_tool(provider: &AIProvider, model: &str) -> bool { model.contains("claude") || provider == &AIProvider::AWSBedrock } +/// Collect the system prompt for providers that take it in a dedicated top-level field +/// (Anthropic's `system`, OpenAI's `instructions`) instead of inline in the message list. +/// +/// Every system message in `messages` is joined, since manual-memory conversations can carry +/// system messages of their own alongside the one the caller prepends from `system_prompt`. +/// `system_prompt` is a fallback used only when `messages` holds no system message, for callers +/// that pass it without prepending it. Only text content survives, so pass just the messages the +/// provider cannot render inline: Anthropic's API takes no system role at all and hands over +/// everything, while OpenAI's accepts system messages inside `input` and hands over only the +/// leading ones. Whatever is passed here must be left out of the message list the provider +/// sends, or the same prompt goes over the wire twice. +pub fn collect_system_prompt( + messages: &[OpenAIMessage], + system_prompt: Option<&str>, +) -> Option { + let from_messages = messages + .iter() + .filter(|message| message.role == "system") + .filter_map(|message| message.content.as_ref().map(extract_text_content)) + .filter(|text| !text.is_empty()) + .collect::>(); + + if from_messages.is_empty() { + system_prompt + .filter(|prompt| !prompt.is_empty()) + .map(str::to_string) + } else { + Some(from_messages.join("\n\n")) + } +} + /// Extract text content from OpenAIContent, joining parts with space if multiple pub fn extract_text_content(content: &OpenAIContent) -> String { match content { @@ -93,6 +184,81 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; + fn message(role: &str, text: &str) -> OpenAIMessage { + OpenAIMessage { + role: role.to_string(), + content: Some(OpenAIContent::Text(text.to_string())), + ..Default::default() + } + } + + #[test] + fn joins_every_system_message() { + let messages = vec![ + message("system", "be helpful"), + message("user", "hi"), + message("system", "be terse"), + ]; + + assert_eq!( + collect_system_prompt(&messages, Some("be helpful")), + Some("be helpful\n\nbe terse".to_string()) + ); + } + + /// A dedicated system field is text-only, so non-text parts cannot be carried over. + #[test] + fn keeps_only_text_parts_of_a_system_message() { + let messages = vec![OpenAIMessage { + role: "system".to_string(), + content: Some(OpenAIContent::Parts(vec![ + ContentPart::Text { text: "be terse".to_string() }, + ContentPart::ImageUrl { + image_url: crate::ai_types::ImageUrlData { + url: "data:image/png;base64,x".to_string(), + }, + }, + ])), + ..Default::default() + }]; + + assert_eq!( + collect_system_prompt(&messages, None), + Some("be terse".to_string()) + ); + } + + /// The argument is a fallback, not an extra source: system messages win outright. + #[test] + fn prefers_system_messages_over_the_argument() { + let messages = vec![message("system", "be terse"), message("user", "hi")]; + + assert_eq!( + collect_system_prompt(&messages, Some("unused fallback")), + Some("be terse".to_string()) + ); + } + + #[test] + fn falls_back_to_the_system_prompt_argument() { + let messages = vec![message("user", "hi")]; + + assert_eq!( + collect_system_prompt(&messages, Some("be helpful")), + Some("be helpful".to_string()) + ); + } + + #[test] + fn treats_empty_prompts_as_absent() { + assert_eq!( + collect_system_prompt(&[message("user", "hi")], Some("")), + None + ); + assert_eq!(collect_system_prompt(&[message("user", "hi")], None), None); + assert_eq!(collect_system_prompt(&[message("system", "")], None), None); + } + /// Regression test for GHSA-5q4v-c4v3-v7wr: `AI_HTTP_CLIENT` must not follow redirects. #[tokio::test] async fn ai_http_client_does_not_follow_redirects() { diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 997879e155..3a2bf83097 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -784,6 +784,11 @@ enum TriggerEdge { runnable_kind: AssetUsageKind, runnable_path: String, }, + Amqp { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, Nats { path: String, runnable_kind: AssetUsageKind, @@ -928,6 +933,9 @@ async fn asset_graph( SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger WHERE workspace_id = $1 UNION ALL + SELECT 'amqp', path, script_path, is_flow FROM amqp_trigger + WHERE workspace_id = $1 + UNION ALL SELECT 'nats', path, script_path, is_flow FROM nats_trigger WHERE workspace_id = $1 UNION ALL @@ -1210,6 +1218,7 @@ async fn asset_graph( "email" => TriggerEdge::Email { path, runnable_kind, runnable_path: script_path }, "kafka" => TriggerEdge::Kafka { path, runnable_kind, runnable_path: script_path }, "mqtt" => TriggerEdge::Mqtt { path, runnable_kind, runnable_path: script_path }, + "amqp" => TriggerEdge::Amqp { path, runnable_kind, runnable_path: script_path }, "nats" => TriggerEdge::Nats { path, runnable_kind, runnable_path: script_path }, "postgres" => TriggerEdge::Postgres { path, runnable_kind, runnable_path: script_path }, "sqs" => TriggerEdge::Sqs { path, runnable_kind, runnable_path: script_path }, diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 25f572d8ae..8120c16605 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -275,6 +275,15 @@ fn scope_restrictions(scopes: Option<&[String]>) -> Option> { (!restrictions.is_empty()).then_some(restrictions) } +/// True when the token carries no real scope restriction — unscoped, an empty scope +/// list, or only `if_jobs:filter_tags:` filters — so it holds the full privileges of +/// its user and can reach any non-job route they are authorized for (mirrors +/// `check_scopes` / `check_route_access`). A `false` result means the token is +/// genuinely scope-restricted. +pub fn is_effectively_unscoped(scopes: Option<&[String]>) -> bool { + scope_restrictions(scopes).is_none() +} + /// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes /// a credential on behalf of `authed`: the resulting credential must never be /// more privileged than the caller's own token. @@ -502,6 +511,89 @@ pub fn build_scope_path_predicate( } } +/// The same `domain:action` path grant as [`build_scope_path_predicate`], decomposed +/// into what a SQL `WHERE` clause needs so the filtering happens IN the query. +/// +/// Filtering in SQL rather than dropping rows after the fetch is mandatory wherever +/// the result is paginated: a post-fetch filter makes a page's size — and any +/// continuation cursor derived from it — reveal the count of rows the caller cannot +/// see. `ScopePathFilter::allows` mirrors the emitted SQL, and a cross-check test +/// pins both to the predicate. +pub enum ScopePathFilter { + /// Unscoped token, or a grant that covers every path: no restriction. + AllowAll, + /// A path is granted iff it equals an `exact` entry or sits at or under a + /// `prefix` (from a `prefix/*` grant, matched on the `/` boundary). Both empty + /// grants nothing. + Restricted { exact: Vec, prefix: Vec }, +} + +impl ScopePathFilter { + /// Whether `path` is granted. Mirrors the SQL a caller builds from this filter, + /// and the matching rule in `resource_matches_pattern`. + pub fn allows(&self, path: &str) -> bool { + match self { + ScopePathFilter::AllowAll => true, + ScopePathFilter::Restricted { exact, prefix } => { + exact.iter().any(|e| e == path) + || prefix.iter().any(|p| { + path == p || path.strip_prefix(p).is_some_and(|r| r.starts_with('/')) + }) + } + } + } +} + +/// Restrictions equivalent to `build_scope_path_predicate(authed, domain, action)`, +/// but pushable into SQL. See [`ScopePathFilter`]. Not for `jobs:run` scopes (whose +/// `kind` dimension this ignores), matching the predicate's path-domain use. +pub fn build_scope_path_filter(authed: &ApiAuthed, domain: &str, action: &str) -> ScopePathFilter { + let (is_scoped_token, parsed): (bool, Vec) = match authed.scopes.as_ref() { + Some(scopes) => { + let mut is_scoped = false; + let parsed = scopes + .iter() + .filter(|s| !s.starts_with("if_jobs:filter_tags:")) + .inspect(|_| is_scoped = true) + .filter_map(|s| ScopeDefinition::from_scope_string(s).ok()) + .collect(); + (is_scoped, parsed) + } + None => (false, Vec::new()), + }; + if !is_scoped_token { + return ScopePathFilter::AllowAll; + } + let mut exact = Vec::new(); + let mut prefix = Vec::new(); + for s in &parsed { + if s.domain != domain { + continue; + } + // `write` covers `read`, mirroring ScopeDefinition::includes' action rule. + if !(s.action == action || (s.action == "write" && action == "read")) { + continue; + } + match &s.resource { + // A domain:action scope with no path part grants every path. + None => return ScopePathFilter::AllowAll, + Some(resources) => { + for r in resources { + // `*` grants every path (resources_match's wildcard short-circuit). + if r == "*" { + return ScopePathFilter::AllowAll; + } + match r.strip_suffix("/*") { + Some(p) => prefix.push(p.to_string()), + None => exact.push(r.clone()), + } + } + } + } + } + ScopePathFilter::Restricted { exact, prefix } +} + pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { let is_devops = is_devops_email(db, email).await?; @@ -1211,6 +1303,48 @@ mod tests { assert!(!allowed("u/alice/bar")); } + // The SQL-pushable filter must grant exactly what the post-fetch predicate does: + // any divergence either leaks/over-grants (filter looser) or hides authorized + // rows (filter tighter). Cross-check both over a matrix of scope sets and paths. + #[test] + fn scope_path_filter_agrees_with_predicate() { + let scope_sets: Vec>> = vec![ + None, + Some(vec![]), + Some(vec!["if_jobs:filter_tags:default"]), + Some(vec!["resources:read"]), + Some(vec!["resources:read:*"]), + Some(vec!["resources:read:u/alice/foo"]), + Some(vec!["resources:read:f/team/*"]), + Some(vec!["resources:write:f/team/*"]), + Some(vec!["resources:read:u/alice/foo,f/team/*"]), + Some(vec!["variables:read:f/team/*"]), + Some(vec!["resources:read:f/team", "resources:read:f/team2/*"]), + ]; + let paths = [ + "u/alice/foo", + "u/alice/foobar", + "u/bob/foo", + "f/team", + "f/team/db", + "f/team/sub/nested", + "f/team2", + "f/other/db", + ]; + for scopes in &scope_sets { + let authed = authed_with_scopes(scopes.clone()); + let predicate = build_scope_path_predicate(&authed, "resources", "read"); + let filter = build_scope_path_filter(&authed, "resources", "read"); + for path in paths { + assert_eq!( + filter.allows(path), + predicate(path), + "mismatch for scopes {scopes:?} path {path}" + ); + } + } + } + fn opt_scopes(scopes: Option>) -> Option> { scopes.map(|v| v.into_iter().map(String::from).collect()) } diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 3c09185650..75cbc104ac 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -240,6 +240,10 @@ pub enum ScopeDomain { // Core resource domains Jobs, Scripts, + /// The `/data_metrics` catalog. Its own domain, NOT an alias of `Scripts`: a + /// `data_metrics:read` token must reach only this route, never the broader + /// `/scripts` routes (some of which do no further scope check). + DataMetrics, Flows, FlowConversations, Apps, @@ -257,6 +261,7 @@ pub enum ScopeDomain { KafkaTriggers, NatsTriggers, MqttTriggers, + AmqpTriggers, SqsTriggers, GcpTriggers, AzureTriggers, @@ -303,6 +308,7 @@ impl ScopeDomain { match self { Self::Jobs => "jobs", Self::Scripts => "scripts", + Self::DataMetrics => "data_metrics", Self::Flows => "flows", Self::FlowConversations => "flow_conversations", Self::Apps => "apps", @@ -318,6 +324,7 @@ impl ScopeDomain { Self::KafkaTriggers => "kafka_triggers", Self::NatsTriggers => "nats_triggers", Self::MqttTriggers => "mqtt_triggers", + Self::AmqpTriggers => "amqp_triggers", Self::SqsTriggers => "sqs_triggers", Self::GcpTriggers => "gcp_triggers", Self::AzureTriggers => "azure_triggers", @@ -355,6 +362,9 @@ impl ScopeDomain { match s { "jobs" | "jobs_u" => Some(Self::Jobs), "scripts" => Some(Self::Scripts), + // A distinct domain, not an alias of `scripts` (see the enum variant): + // a `data_metrics:read` token must not reach the broader /scripts routes. + "data_metrics" => Some(Self::DataMetrics), "flows" => Some(Self::Flows), "flow_conversations" => Some(Self::FlowConversations), "apps" | "apps_u" => Some(Self::Apps), @@ -370,6 +380,7 @@ impl ScopeDomain { "kafka_triggers" => Some(Self::KafkaTriggers), "nats_triggers" => Some(Self::NatsTriggers), "mqtt_triggers" => Some(Self::MqttTriggers), + "amqp_triggers" => Some(Self::AmqpTriggers), "sqs_triggers" => Some(Self::SqsTriggers), "gcp_triggers" => Some(Self::GcpTriggers), "azure_triggers" => Some(Self::AzureTriggers), @@ -987,6 +998,24 @@ mod tests { assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "DELETE").is_err()); } + #[test] + fn data_metrics_is_its_own_domain_not_a_scripts_alias() { + // `data_metrics` must be a distinct domain: a token scoped to it must reach + // only the data_metrics route, never the broader /scripts routes (some of + // which do no further scope check). Regression for a privilege escalation. + assert_eq!( + ScopeDomain::from_str("data_metrics"), + Some(ScopeDomain::DataMetrics) + ); + let dm = vec!["data_metrics:read".to_string()]; + assert!(check_route_access(&dm, "/api/w/test/data_metrics/list", "GET").is_ok()); + assert!(check_route_access(&dm, "/api/w/test/scripts/list", "GET").is_err()); + assert!(check_route_access(&dm, "/api/w/test/scripts/raw/h/abc.ts", "GET").is_err()); + // Conversely a scripts token does not reach the data_metrics route. + let sc = vec!["scripts:read".to_string()]; + assert!(check_route_access(&sc, "/api/w/test/data_metrics/list", "GET").is_err()); + } + #[test] fn test_new_domain_parsing() { // Test that new domains are properly parsed diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs index 016e72be91..35816dfaa8 100644 --- a/backend/windmill-api-embeddings/src/lib.rs +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -56,6 +56,10 @@ lazy_static::lazy_static! { pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); + // On a failed init/refresh we retry after this short interval instead of the + // full pulling interval, so a transient startup error doesn't leave the + // embeddings DB uninitialized for a whole day. + pub static ref HUB_EMBEDDINGS_RETRY_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_RETRY_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(60); } #[cfg(feature = "embedding")] @@ -112,6 +116,26 @@ pub struct ResourceTypeResult { score: f32, schema: Option, } + +/// Drop results whose score falls more than `max_relative_drop` below the best +/// match, so a strong hit isn't diluted by weakly-related entries that merely +/// clear the similarity floor. Expects `results` sorted by descending score and +/// a positive top score (guaranteed by the caller's similarity threshold). +#[cfg(feature = "embedding")] +fn trim_to_top_score( + results: Vec, + max_relative_drop: f32, + score: impl Fn(&T) -> f32, +) -> Vec { + if results.len() <= 1 { + return results; + } + let top_score = score(&results[0]); + results + .into_iter() + .take_while(|r| (top_score - score(r)) / top_score <= max_relative_drop) + .collect() +} #[cfg(feature = "embedding")] async fn query_resource_types( Query(query): Query, @@ -511,15 +535,7 @@ impl EmbeddingsDb { }) .collect(); - let mut results = results?; - - if results.len() > 1 { - let top_score = results[0].score; - results = results - .into_iter() - .take_while(|r| (top_score - r.score) / top_score <= 0.05) - .collect(); - } + let results = trim_to_top_score(results?, 0.05, |r| r.score); Ok(results) } @@ -560,7 +576,7 @@ impl EmbeddingsDb { Some(0.75), ); - let results: Result<_> = results + let results: Result> = results .iter() .map(|r| { let metadata = r @@ -582,7 +598,9 @@ impl EmbeddingsDb { }) .collect(); - results + let results = trim_to_top_score(results?, 0.05, |r| r.score); + + Ok(results) } } @@ -601,42 +619,67 @@ pub fn load_embeddings_db(db: &Pool) -> () { if !disable_embedding { let db_clone = db.clone(); tokio::spawn(async move { - let model_instance = ModelInstance::new().await; - if let Ok(model_instance) = model_instance { - let mut model_instance_lock = MODEL_INSTANCE.write().await; - *model_instance_lock = Some(Arc::new(model_instance)); - drop(model_instance_lock); - loop { - update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs( - *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, - )) - .await; + // Keep retrying model init: a transient failure here must not + // permanently disable embeddings until the next process restart. + // Backoff decays to the pulling interval so an environment where it + // can never succeed (e.g. air-gapped, embeddings left enabled) + // settles into ~1 attempt/interval rather than a tight error loop. + let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + loop { + match ModelInstance::new().await { + Ok(model_instance) => { + let mut model_instance_lock = MODEL_INSTANCE.write().await; + *model_instance_lock = Some(Arc::new(model_instance)); + break; + } + Err(e) => { + tracing::error!( + "Failed to initialize model instance: {}. Retrying in {}s...", + e, + backoff_secs + ); + tokio::time::sleep(std::time::Duration::from_secs(backoff_secs)).await; + backoff_secs = backoff_secs + .saturating_mul(2) + .min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS); + } } - } else { - tracing::error!( - "Failed to initialize model instance: {}", - model_instance.err().unwrap() - ); + } + let mut backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + loop { + let sleep_secs = if update_embeddings_db(&db_clone).await { + backoff_secs = *HUB_EMBEDDINGS_RETRY_INTERVAL_SECS; + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS + } else { + let secs = backoff_secs; + backoff_secs = backoff_secs + .saturating_mul(2) + .min(*HUB_EMBEDDINGS_PULLING_INTERVAL_SECS); + secs + }; + tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await; } }); } } #[cfg(feature = "embedding")] -pub async fn update_embeddings_db(db: &Pool) -> () { +pub async fn update_embeddings_db(db: &Pool) -> bool { if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { tracing::info!("Creating embeddings DB..."); let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; if let Err(e) = new_embeddings_db.as_ref() { tracing::error!("Failed to create embeddings db: {}", e); + false } else { let mut embeddings_db = EMBEDDINGS_DB.write().await; *embeddings_db = new_embeddings_db.ok(); tracing::info!("Created embeddings DB"); + true } } else { tracing::error!("Could not update embeddings DB, model instance not initialized"); + false } } @@ -659,3 +702,31 @@ pub fn workspaced_service() -> Router { pub fn global_service() -> Router { Router::new() } + +#[cfg(all(test, feature = "embedding"))] +mod tests { + use super::trim_to_top_score; + + #[test] + fn trims_scores_more_than_5pct_below_top() { + // top=1.0, cutoff at 0.95: 0.96 stays (0.04 drop), 0.93 is the first + // beyond the cutoff so take_while stops there and drops the tail. + let kept = trim_to_top_score(vec![1.0f32, 0.97, 0.96, 0.93, 0.9], 0.05, |s| *s); + assert_eq!(kept, vec![1.0, 0.97, 0.96]); + } + + #[test] + fn keeps_all_when_tightly_clustered() { + let kept = trim_to_top_score(vec![0.9f32, 0.89, 0.88], 0.05, |s| *s); + assert_eq!(kept, vec![0.9, 0.89, 0.88]); + } + + #[test] + fn passes_through_zero_or_one_result() { + assert_eq!( + trim_to_top_score(Vec::::new(), 0.05, |s| *s), + Vec::::new() + ); + assert_eq!(trim_to_top_score(vec![0.42f32], 0.05, |s| *s), vec![0.42]); + } +} diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index a9376d02ed..97e6999e23 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -47,7 +47,7 @@ use windmill_common::HUB_BASE_URL; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error, JsonResult, Result}, - flows::{Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, + flows::{EditFlow, Flow, FlowWithStarred, ListFlowQuery, ListableFlow, NewFlow}, jobs::JobPayload, schedule::Schedule, utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, @@ -101,11 +101,7 @@ async fn list_search_flows( Path(w_id): Path, Extension(user_db): Extension, ) -> JsonResult> { - #[cfg(feature = "enterprise")] let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; let mut tx = user_db.begin(&authed).await?; let allowed = build_scope_path_predicate(&authed, "flows", "read"); @@ -531,6 +527,11 @@ async fn create_flow( } check_scopes(&authed, || format!("flows:write:{}", nf.path))?; + // A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly. + // (The concurrency settings inside the flow value are normalized on deserialization; see + // ConcurrencySettings.) Runtime guards also protect already-stored rows. + nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout); + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -1007,7 +1008,7 @@ async fn update_flow( Extension(db): Extension, Extension(webhook): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, - Json(nf): Json, + Json(ef): Json, ) -> Result { if authed.is_operator { return Err(Error::NotAuthorized( @@ -1015,6 +1016,10 @@ async fn update_flow( )); } let flow_path = flow_path.to_path(); + // The URL identifies the flow being updated; the body path is only needed to rename. + let mut nf = ef.into_new_flow(flow_path); + // A `<= 0` flow timeout is "unset", not a 0-second limit (see create_flow). + nf.timeout = windmill_common::runnable_settings::none_if_non_positive(nf.timeout); check_scopes(&authed, || format!("flows:write:{}", flow_path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index 67997bc965..8736f7b60f 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -15,7 +15,10 @@ use axum::{ }; use lazy_static::lazy_static; use regex::Regex; -use windmill_api_auth::{check_scopes, ApiAuthed, AuthCache, Tokened}; +use windmill_api_auth::{ + build_scope_path_filter, build_scope_path_predicate, check_scopes, ApiAuthed, AuthCache, + ScopePathFilter, Tokened, +}; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::DB; @@ -109,6 +112,7 @@ async fn list_folders( let (per_page, offset) = paginate(pagination); let mut tx = user_db.begin(&authed).await?; + let allowed = build_scope_path_predicate(&authed, "folders", "read"); let rows = sqlx::query_as!( Folder, "SELECT workspace_id, name, display_name, owners, extra_perms, summary, created_by, edited_at, default_permissioned_as, labels FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", @@ -117,7 +121,10 @@ async fn list_folders( offset as i64 ) .fetch_all(&mut *tx) - .await?; + .await? + .into_iter() + .filter(|r| allowed(&format!("f/{}", r.name))) + .collect::>(); tx.commit().await?; Ok(Json(rows)) @@ -131,14 +138,43 @@ async fn list_foldernames( let (per_page, offset) = paginate(pagination); let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_scalar!( - "SELECT name FROM folder WHERE workspace_id = $1 ORDER BY name asc LIMIT $2 OFFSET $3", - w_id, - per_page as i64, - offset as i64 - ) - .fetch_all(&mut *tx) - .await?; + // Push the token's scope grant into the query so LIMIT/OFFSET page over the + // AUTHORIZED folders — the returned count then reflects the authorized set, so a + // paginating caller can rely on `< per_page` meaning exhaustion. (Filtering after the + // LIMIT would let a page return fewer than per_page while authorized folders remain + // on later DB pages, stopping such a caller early.) + let mut sql = String::from("SELECT name FROM folder WHERE workspace_id = $1"); + let restricted = match build_scope_path_filter(&authed, "folders", "read") { + ScopePathFilter::AllowAll => None, + ScopePathFilter::Restricted { exact, prefix } => { + // A prefix grant also authorizes the folder at the prefix itself. + let mut eq = exact; + eq.append(&mut prefix.clone()); + let like: Vec = prefix + .iter() + .map(|p| { + format!( + "{}/%", + p.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") + ) + }) + .collect(); + sql.push_str(" AND (('f/' || name) = ANY($4) OR ('f/' || name) LIKE ANY($5))"); + Some((eq, like)) + } + }; + sql.push_str(" ORDER BY name asc LIMIT $2 OFFSET $3"); + + let mut query = sqlx::query_scalar::<_, String>(&sql) + .bind(&w_id) + .bind(per_page as i64) + .bind(offset as i64); + if let Some((eq, like)) = restricted { + query = query.bind(eq).bind(like); + } + let rows = query.fetch_all(&mut *tx).await?; tx.commit().await?; @@ -235,6 +271,7 @@ async fn create_folder( Path(w_id): Path, Json(mut ng): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Folder creation")?; if let Some(labels) = ng.labels.as_mut() { dedup_labels(labels); } @@ -410,6 +447,12 @@ async fn update_folder( return Err(Error::PermissionDenied(msg)); } + // update_folder can also grant permissions (owners / extra_perms / default_permissioned_as), + // so it is a sharing path and must honor the demo-workspace sharing restriction. + if ng.owners.is_some() || ng.extra_perms.is_some() || ng.default_permissioned_as.is_some() { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; + } + let mut sqlb = SqlBuilder::update_table("folder"); sqlb.and_where_eq("name", "?".bind(&name)); sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); @@ -812,6 +855,7 @@ async fn add_owner( Path((w_id, name)): Path<(String, String)>, Json(Owner { owner, .. }): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; let mut tx = user_db.begin(&authed).await?; not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; @@ -876,6 +920,13 @@ async fn remove_owner( Path((w_id, name)): Path<(String, String)>, Json(Owner { owner, write }): Json, ) -> Result { + // remove_owner with a `write` value is a grant path: it jsonb_set's the owner's + // permission level into extra_perms (only write=None is a pure revoke), so the + // demo-workspace sharing restriction must apply when a level is being set. + if write.is_some() { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; + } + let mut tx = user_db.begin(&authed).await?; not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?; diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 663644e8e0..5a7af05cbe 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -46,13 +46,13 @@ fn audit_action_prefix_for_acl_kind(kind: &str) -> Option<&'static str> { "variable" => Some("variables"), "schedule" => Some("schedules"), "http_trigger" | "websocket_trigger" | "kafka_trigger" | "nats_trigger" - | "postgres_trigger" | "mqtt_trigger" | "gcp_trigger" | "azure_trigger" | "sqs_trigger" - | "email_trigger" => Some("triggers"), + | "postgres_trigger" | "mqtt_trigger" | "amqp_trigger" | "gcp_trigger" + | "azure_trigger" | "sqs_trigger" | "email_trigger" => Some("triggers"), _ => None, } } -const KINDS: [&str; 20] = [ +const KINDS: [&str; 21] = [ "script", "group_", "resource", @@ -68,6 +68,7 @@ const KINDS: [&str; 20] = [ "nats_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "gcp_trigger", "azure_trigger", "sqs_trigger", @@ -95,6 +96,7 @@ async fn add_granular_acl( Path((w_id, path)): Path<(String, StripPath)>, Json(GranularAcl { owner, write }): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Sharing")?; let path = path.to_path(); let (kind, path) = path diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index 263daad624..d2a6c7d16e 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -234,6 +234,7 @@ async fn create_group( Path(w_id): Path, Json(ng): Json, ) -> Result { + crate::check_demo_workspace_restriction(&authed, &w_id, "Group creation")?; let mut tx = user_db.begin(&authed).await?; check_name_conflict(&mut tx, &w_id, &ng.name).await?; diff --git a/backend/windmill-api-groups/src/lib.rs b/backend/windmill-api-groups/src/lib.rs index 75b9349105..7abfa426ee 100644 --- a/backend/windmill-api-groups/src/lib.rs +++ b/backend/windmill-api-groups/src/lib.rs @@ -2,3 +2,23 @@ pub mod folder_history; pub mod folders; pub mod granular_acls; pub mod groups; + +use windmill_api_auth::ApiAuthed; +use windmill_common::{error::Error, worker::CLOUD_HOSTED}; + +/// The public demo workspace on the managed cloud is kept clean and consistent by +/// restricting folder creation, item sharing, and group creation for non-admins. +/// `action` is a short noun phrase completing "… is disabled …" (e.g. +/// "Folder creation", "Sharing"). Returns `Err(BadRequest)` when the caller is blocked. +pub fn check_demo_workspace_restriction( + authed: &ApiAuthed, + w_id: &str, + action: &str, +) -> Result<(), Error> { + if *CLOUD_HOSTED && w_id == "demo" && !authed.is_admin { + return Err(Error::BadRequest(format!( + "{action} is disabled in the demo workspace. Create your own workspace to keep the demo clean and consistent." + ))); + } + Ok(()) +} diff --git a/backend/windmill-api-inputs/src/lib.rs b/backend/windmill-api-inputs/src/lib.rs index 348b434eea..8e33789c3d 100644 --- a/backend/windmill-api-inputs/src/lib.rs +++ b/backend/windmill-api-inputs/src/lib.rs @@ -73,14 +73,6 @@ impl Display for RunnableType { } impl RunnableType { - fn job_kind(&self) -> JobKind { - match self { - RunnableType::ScriptHash => JobKind::Script, - RunnableType::ScriptPath => JobKind::Script, - RunnableType::FlowPath => JobKind::Flow, - } - } - fn column_name(&self) -> &'static str { match self { RunnableType::ScriptHash => "runnable_id", @@ -159,6 +151,28 @@ async fn get_input_history( "AND parent_job IS NULL" }; + // A scheduled runnable with a dynamic-skip handler (or a scheduled script with + // native retry) is wrapped in a synthetic `singlestepflow`, so its runs land under + // that kind rather than `flow`/`script` (see windmill-queue schedule.rs). Such a + // wrapper holds either a script or a flow, and the two may share a runnable_path, so + // match only the wrapped kind that belongs to the queried runnable. The wrapped type + // lives in raw_flow.modules[id in ('a','main')].value.type (mirrors the projection in + // windmill-api jobs.rs). runnable_id is NULL on these rows, so ScriptHash never matches. + let singlestepflow_filter = match r.runnable_type { + RunnableType::FlowPath => { + "AND (kind <> 'singlestepflow' OR EXISTS (\ + SELECT 1 FROM jsonb_array_elements(v2_job.raw_flow->'modules') m \ + WHERE m->>'id' IN ('a', 'main') AND m->'value'->>'type' = 'flow'))" + } + RunnableType::ScriptPath => { + "AND (kind <> 'singlestepflow' OR EXISTS (\ + SELECT 1 FROM jsonb_array_elements(v2_job.raw_flow->'modules') m \ + WHERE m->>'id' IN ('a', 'main') \ + AND COALESCE(m->'value'->>'type', 'script') <> 'flow'))" + } + RunnableType::ScriptHash => "", + }; + // Two-step approach: first fetch 2*(per_page+offset) rows using created_at ordering // (which leverages the ix_job_root_job_index_by_path_2 index on v2_job), then sort // the small result set by completed_at. This works because created_at and completed_at @@ -172,7 +186,7 @@ async fn get_input_history( kind IN ('preview', 'flowpreview') as is_preview \ FROM v2_job JOIN v2_job_completed USING (id) \ WHERE v2_job.workspace_id = $3 AND {} = $1 AND kind = any($2) \ - AND v2_job.script_entrypoint_override IS NULL \ + AND v2_job.script_entrypoint_override IS NULL {singlestepflow_filter} \ {args_query} AND v2_job_completed.status != 'skipped' {include_non_root} \ ORDER BY v2_job.created_at DESC LIMIT $4\ ) t ORDER BY completed_at DESC LIMIT $5 OFFSET $6", @@ -186,14 +200,32 @@ async fn get_input_history( _ => query.bind(&r.runnable_id), }; - let job_kinds = match r.runnable_type.job_kind() { - kind @ JobKind::Script if g.include_preview.unwrap_or(false) => { - vec![kind, JobKind::Preview] + // Include SingleStepFlow so scheduled runs surface (see `singlestepflow_filter` + // above, which restricts it to the wrapped kind matching the runnable). ScriptHash + // is omitted: those wrappers carry no runnable_id, so it can never match. + let include_preview = g.include_preview.unwrap_or(false); + let job_kinds = match r.runnable_type { + RunnableType::ScriptHash => { + let mut kinds = vec![JobKind::Script]; + if include_preview { + kinds.push(JobKind::Preview); + } + kinds } - kind @ JobKind::Flow if g.include_preview.unwrap_or(false) => { - vec![kind, JobKind::FlowPreview] + RunnableType::ScriptPath => { + let mut kinds = vec![JobKind::Script, JobKind::SingleStepFlow]; + if include_preview { + kinds.push(JobKind::Preview); + } + kinds + } + RunnableType::FlowPath => { + let mut kinds = vec![JobKind::Flow, JobKind::SingleStepFlow]; + if include_preview { + kinds.push(JobKind::FlowPreview); + } + kinds } - kind => vec![kind], }; let rows = query @@ -246,10 +278,11 @@ async fn get_args_from_history_or_saved_input( let result_o = if let Some(input) = g.input { if input { sqlx::query_scalar!( - "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2 AND (is_public IS true OR created_by = $4)", job_or_input_id, w_id, - g.allow_large.unwrap_or(true) + g.allow_large.unwrap_or(true), + authed.username ) .fetch_optional(&mut *tx) .await? @@ -267,10 +300,11 @@ async fn get_args_from_history_or_saved_input( sqlx::query_scalar!( "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM v2_job WHERE id = $1 AND workspace_id = $2 UNION ALL - SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2", + SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2 AND (is_public IS true OR created_by = $4)", job_or_input_id, w_id, - g.allow_large.unwrap_or(true) + g.allow_large.unwrap_or(true), + authed.username ) .fetch_optional(&mut *tx) .await? diff --git a/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql b/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql new file mode 100644 index 0000000000..bd1fd11788 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fixtures/fork_member_grant.sql @@ -0,0 +1,33 @@ +-- `test2` (a non-admin developer of test-workspace) has forked it. The fork's `usr` row copies the +-- non-admin role they hold in the parent, which is the situation the fork-creator grant exists for. +INSERT INTO workspace (id, name, owner, parent_workspace_id) VALUES + ('wm-fork-test', 'fork of test-workspace', 'test2@windmill.dev', 'test-workspace'); + +INSERT INTO workspace_settings (workspace_id) VALUES ('wm-fork-test'); + +INSERT INTO workspace_key (workspace_id, kind, key) VALUES ('wm-fork-test', 'cloud', 'test-key'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES + ('wm-fork-test', 'all', 'All users', '{}'); + +INSERT INTO usr (workspace_id, email, username, is_admin, role) VALUES + ('wm-fork-test', 'test2@windmill.dev', 'test-user-2', false, 'User'), + -- An admin of the fork, whom its creator must not be able to remove. + ('wm-fork-test', 'test@windmill.dev', 'test-user', true, 'Admin'); + +-- An operator of the parent: the eligibility bar for being added to the fork is developer-or-above +-- there, so this user must be rejected. +INSERT INTO password (email, password_hash, login_type, super_admin, verified, name, username) + VALUES ('test4@windmill.dev', 'not-a-real-hash', 'password', false, true, 'Test User 4', 'test-user-4'); + +INSERT INTO usr (workspace_id, email, username, is_admin, operator, role) VALUES + ('test-workspace', 'test4@windmill.dev', 'test-user-4', false, true, 'Operator'); + +-- add_user resolves the instance-wide username from `password`. +UPDATE password SET username = 'test-user-2' WHERE email = 'test2@windmill.dev'; +UPDATE password SET username = 'test-user-3' WHERE email = 'test3@windmill.dev'; + +-- With automated username creation off, `add_user` takes the username from the caller. That branch +-- is what the fork creator must not be able to steer, so the tests run against it. +INSERT INTO global_settings (name, value) VALUES ('automate_username_creation', 'false'::jsonb) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; diff --git a/backend/windmill-api-integration-tests/tests/fork_member_grant.rs b/backend/windmill-api-integration-tests/tests/fork_member_grant.rs new file mode 100644 index 0000000000..8789391854 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fork_member_grant.rs @@ -0,0 +1,187 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +/// `test2` created the fork `wm-fork-test` but is only a developer in it. +const FORK_OWNER_TOKEN: &str = "SECRET_TOKEN_2"; +/// `test3` is a developer of the parent and of the fork, but created neither. +const FORK_MEMBER_TOKEN: &str = "SECRET_TOKEN_3"; + +fn as_user(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {token}")) +} + +async fn add_user( + port: u16, + w_id: &str, + token: &str, + body: serde_json::Value, +) -> reqwest::Response { + as_user( + client().post(format!( + "http://localhost:{port}/api/w/{w_id}/workspaces/add_user" + )), + token, + ) + .json(&body) + .send() + .await + .unwrap() +} + +fn developer(email: &str) -> serde_json::Value { + json!({ "email": email, "is_admin": false, "operator": false }) +} + +/// The creator of a fork may manage developers on it without being an admin of it, and may do +/// nothing beyond that. A fork clones its parent wholesale (secrets included), so each of these +/// bounds is what keeps the grant from becoming a way for any developer to widen access to the +/// parent's data or to mint an admin. +#[sqlx::test(migrations = "../migrations", fixtures("base", "fork_member_grant"))] +async fn test_fork_creator_can_only_manage_developers_on_their_fork( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Seating an eligible member on someone else's username is what would hand them that user's + // cloned private assets. `test-user-4` is the target that matters: a fork clones the parent's + // `u/test-user-4/` scripts, variables and secrets but not their membership, so the username is + // free of the unique constraint on `usr` and the squat would otherwise land. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + json!({ "email": "test3@windmill.dev", "username": "test-user-4", "is_admin": false, "operator": false }), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot choose the username a member joins under" + ); + + // The creator adds a developer of the parent as a developer of their fork. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("test3@windmill.dev"), + ) + .await; + assert_eq!(resp.status(), 201, "fork creator can add a developer"); + + // They join under the username they hold in the parent, which is the `u/` namespace the fork + // cloned for them. + let username: String = sqlx::query_scalar( + "SELECT username FROM usr WHERE workspace_id = 'wm-fork-test' AND email = 'test3@windmill.dev'", + ) + .fetch_one(&db) + .await?; + assert_eq!( + username, "test-user-3", + "added member keeps their parent username" + ); + + // ... but never as an admin. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + json!({ "email": "test4@windmill.dev", "is_admin": true, "operator": false }), + ) + .await; + assert_eq!(resp.status(), 403, "fork creator cannot add an admin"); + + // ... nor anyone who is only an operator of the parent, which would widen their access. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot add an operator of the parent" + ); + + // ... nor anyone from outside the parent workspace. + let resp = add_user( + port, + "wm-fork-test", + FORK_OWNER_TOKEN, + developer("outsider@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator cannot add a non-member of the parent" + ); + + // The grant covers the fork alone, not the workspace it was forked from. + let resp = add_user( + port, + "test-workspace", + FORK_OWNER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "fork creator gains nothing on the parent workspace" + ); + + // ... and belongs to the creator, not to every member of the fork. + let resp = add_user( + port, + "wm-fork-test", + FORK_MEMBER_TOKEN, + developer("test4@windmill.dev"), + ) + .await; + assert_eq!( + resp.status(), + 403, + "a fork member who did not create it gains nothing" + ); + + // Removing is the counterpart of adding: allowed for the developer they just added... + let resp = as_user( + client().delete(format!( + "http://localhost:{port}/api/w/wm-fork-test/users/delete/test-user-3" + )), + FORK_OWNER_TOKEN, + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "fork creator can remove a developer"); + + // ... but not for an admin of the fork. + let resp = as_user( + client().delete(format!( + "http://localhost:{port}/api/w/wm-fork-test/users/delete/test-user" + )), + FORK_OWNER_TOKEN, + ) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 403, + "fork creator cannot remove an admin of the fork" + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/inputs.rs b/backend/windmill-api-integration-tests/tests/inputs.rs index 6f806efdb5..a3f0610e55 100644 --- a/backend/windmill-api-integration-tests/tests/inputs.rs +++ b/backend/windmill-api-integration-tests/tests/inputs.rs @@ -74,3 +74,94 @@ async fn test_inputs_endpoints(db: Pool) -> anyhow::Result<()> { Ok(()) } + +// A scheduled runnable with a dynamic-skip handler (or a scheduled script with native +// retry) runs as a `singlestepflow`, not `flow`/`script` (see windmill-queue schedule.rs). +// Both a flow's and a script's history must surface their own singlestepflow runs, but a +// script and flow may share a path, so each side must match only the wrapped kind that +// belongs to it — the other must not leak in. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_input_history_singlestepflow_flow_vs_script( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + // Both rows share runnable_path 'f/test/scheduled'. + let flow_job = insert_singlestepflow(&db, "flow").await?; + let script_job = insert_singlestepflow(&db, "script").await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/inputs"); + + let history_ids = |runnable_type: &'static str| { + let base = base.clone(); + async move { + let resp = authed(client().get(format!( + "{base}/history?runnable_id=f/test/scheduled&runnable_type={runnable_type}" + ))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /inputs/history"); + let inputs: Vec = serde_json::from_str(&body)?; + anyhow::Ok( + inputs + .iter() + .filter_map(|i| i.get("id").and_then(|v| v.as_str()).map(String::from)) + .collect::>(), + ) + } + }; + + let flow_hist = history_ids("FlowPath").await?; + assert!( + flow_hist.contains(&flow_job.to_string()), + "flow-wrapped singlestepflow missing from flow history: {flow_hist:?}", + ); + assert!( + !flow_hist.contains(&script_job.to_string()), + "script-wrapped singlestepflow leaked into flow history: {flow_hist:?}", + ); + + let script_hist = history_ids("ScriptPath").await?; + assert!( + script_hist.contains(&script_job.to_string()), + "script-wrapped singlestepflow missing from script history: {script_hist:?}", + ); + assert!( + !script_hist.contains(&flow_job.to_string()), + "flow-wrapped singlestepflow leaked into script history: {script_hist:?}", + ); + + Ok(()) +} + +// Insert a completed root singlestepflow at path f/test/scheduled wrapping `wrapped_type` +// ('flow' or 'script') as its single module — mirrors the schedule.rs wrapper shape. +async fn insert_singlestepflow( + db: &Pool, + wrapped_type: &str, +) -> anyhow::Result { + let id = uuid::Uuid::new_v4(); + let raw_flow = json!({ "modules": [{ "id": "a", "value": { "type": wrapped_type } }] }); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, tag, created_by, permissioned_as, \ + permissioned_as_email, kind, runnable_path, raw_flow, same_worker, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'flow', 'test-user', 'u/test-user', \ + 'test@windmill.dev', 'singlestepflow', 'f/test/scheduled', $2, false, true)", + ) + .bind(id) + .bind(sqlx::types::Json(&raw_flow)) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_completed (id, workspace_id, duration_ms, deleted, status) \ + VALUES ($1, 'test-workspace', 1, false, 'success')", + ) + .bind(id) + .execute(db) + .await?; + Ok(id) +} diff --git a/backend/windmill-api-integration-tests/tests/resources.rs b/backend/windmill-api-integration-tests/tests/resources.rs index cc78056176..3cc59be473 100644 --- a/backend/windmill-api-integration-tests/tests/resources.rs +++ b/backend/windmill-api-integration-tests/tests/resources.rs @@ -275,6 +275,19 @@ async fn test_resource_endpoints(db: Pool) -> anyhow::Result<()> { let body = resp.json::().await?; assert_eq!(body["description"], "Updated description"); + // --- update (resource_type) --- + // An update that only changes resource_type must persist it. + let resp = authed(client().post(resource_url(port, "update", "u/test-user/new_resource"))) + .json(&json!({"resource_type": "mcp_server"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + + let resp = authed_get(port, "get", "u/test-user/new_resource").await; + let body = resp.json::().await?; + assert_eq!(body["resource_type"], "mcp_server"); + // --- update_value --- let resp = authed(client().post(resource_url( port, diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index 92f179c39b..a681fd982e 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -457,6 +457,20 @@ async fn test_workspace_endpoints(db: Pool) -> anyhow::Result<()> { resp.status() ); + // --- git_sync_deploy_mode (response shape + default when no git-sync configured) --- + let resp = authed(client().get(format!("{base}/git_sync_deploy_mode"))) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 200, + "git_sync_deploy_mode: unexpected status" + ); + let mode = resp.json::().await?; + assert_eq!(mode["configured"], json!(false)); + assert_eq!(mode["deploy_on_push"], json!(false)); + // --- update_operator_settings --- let resp = authed(client().post(format!("{base}/operator_settings"))) .json(&json!({})) @@ -887,6 +901,64 @@ async fn test_get_copilot_info_ignores_empty_instance_ai_row( Ok(()) } +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_error_handler_instance_alerts_fallback(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/workspaces"); + + let stored = || async { + sqlx::query_scalar!( + "SELECT error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = 'test-workspace'" + ) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "enable: {}", resp.text().await?); + assert!(stored().await?); + + // A client that predates the setting (the CLI pushing settings.yaml) omits the field and + // must not silently turn it back off. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "extra_args": null})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "omitted: {}", resp.text().await?); + assert!(stored().await?); + + sqlx::query!( + "UPDATE workspace SET parent_workspace_id = 'test-workspace' WHERE id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": true})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400, "fork must be rejected"); + + // The settings page stops offering the option once the workspace is a fork, so its next save + // sends `false`: that must go through rather than lock the whole error handler behind a 400. + let resp = authed(client().post(format!("{base}/edit_error_handler"))) + .json(&json!({"path": null, "fallback_to_instance_alerts": false})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "disable on fork: {}", resp.text().await?); + assert!(!stored().await?); + + Ok(()) +} + #[sqlx::test(migrations = "../migrations", fixtures("base"))] async fn test_get_imports(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs index 8303788b8a..3f559b4c25 100644 --- a/backend/windmill-api-jobs/src/concurrency_groups.rs +++ b/backend/windmill-api-jobs/src/concurrency_groups.rs @@ -357,6 +357,7 @@ async fn get_concurrent_intervals( } async fn get_concurrency_key( + _authed: ApiAuthed, Extension(db): Extension, Path(job_id): Path, ) -> JsonResult> { diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index 0b188c0fa0..4f481fbd54 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -19,6 +19,7 @@ use windmill_common::{ jobs::{is_safe_log_file_path, JobKind, JobStatus, JobTriggerKind}, scripts::ScriptLang, utils::{paginate, paginate_without_limits, require_admin, Pagination}, + worker::CLOUD_HOSTED, }; use windmill_api_auth::ApiAuthed; @@ -435,6 +436,15 @@ pub async fn import_queued_jobs( ) -> error::Result { require_admin(authed.is_admin, &authed.username)?; + // Self-hosted migration/restore tool. It writes queue rows straight to the DB, bypassing the + // push path and every admission check that lives there, so it has no place on multi-tenant + // cloud where those checks are what keep one workspace from swamping the shared pool. + if *CLOUD_HOSTED { + return Err(error::Error::BadRequest( + "Importing queued jobs is not available on the cloud".to_string(), + )); + } + let mut tx = user_db.begin(&authed).await?; for job in jobs { diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 573c47e355..e9ca4d54fe 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -16,7 +16,9 @@ use serde::{Deserialize, Serialize}; use sql_builder::{prelude::Bind, SqlBuilder}; use sqlx::{Postgres, Transaction}; use std::str::FromStr; -use windmill_api_auth::{check_scopes, maybe_refresh_folders, require_super_admin, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_super_admin, ApiAuthed, +}; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::DB; @@ -911,6 +913,9 @@ async fn list_schedule( } } + let allowed = build_scope_path_predicate(&authed, "schedules", "read"); + rows.retain(|r| allowed(&r.path)); + Ok(Json(rows)) } @@ -958,7 +963,10 @@ async fn list_schedule_with_jobs( .fetch_all(&mut *tx) .await?; tx.commit().await?; - Ok(Json(rows)) + let allowed = build_scope_path_predicate(&authed, "schedules", "read"); + Ok(Json( + rows.into_iter().filter(|r| allowed(&r.path)).collect(), + )) } // SELECT id, title AS item_title, t.tag_array diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index fd9899ea10..c2f023ce5a 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -157,12 +157,8 @@ async fn list_search_scripts( Extension(user_db): Extension, ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] let n = 10000; - #[cfg(not(feature = "enterprise"))] - let n = 10; - let allowed = build_scope_path_predicate(&authed, "scripts", "read"); let rows = sqlx::query_as!( SearchScript, @@ -417,11 +413,19 @@ async fn list_scripts( // A draft-only pipeline node (`// pipeline`) has no deployed row to carry // auto_kind, so compute it from the draft content — mirroring the create // path — so the home page folds it into its pipeline like a deployed member. + // Otherwise fall back to the `auto_kind` the frontend saved into the draft + // (e.g. `lib` for scripts without a `main`); the content-derived pipeline + // annotation keeps priority since it mirrors the deploy-time computation. let auto_kind = v .get("content") .and_then(|s| s.as_str()) .filter(|c| parse_pipeline_annotations(c).in_pipeline) - .map(|_| "pipeline".to_string()); + .map(|_| "pipeline".to_string()) + .or_else(|| { + v.get("auto_kind") + .and_then(|s| s.as_str()) + .map(|s| s.to_string()) + }); rows.push(ListableScript { hash: ScriptHash(0), path: row.path, @@ -910,6 +914,13 @@ async fn create_script_internal<'c>( } check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + // Normalize positive-only settings so a `<= 0` value (e.g. a CLI-pushed `0`) persists as + // disabled rather than as a zero-slot concurrency cap or a 0-second timeout. Deserialization + // already normalizes the concurrency fields; re-applying here also covers `timeout` and any + // NewScript built in-process rather than from a request body. + ns.timeout = windmill_common::runnable_settings::none_if_non_positive(ns.timeout); + ns.concurrency_settings = ns.concurrency_settings.normalized(); + guard_script_from_debounce_data(&ns).await?; let codebase = ns.codebase.as_ref(); @@ -1339,6 +1350,18 @@ async fn create_script_internal<'c>( malformed_data_tests ); } + let (malformed_measures, malformed_dimensions) = + windmill_parser::asset_parser::count_malformed_metric_annotations(&ns.content); + if malformed_measures > 0 || malformed_dimensions > 0 { + tracing::warn!( + "script {}: {} `// measure` and {} `// dimension` line(s) are malformed and were \ + dropped. Fix the syntax (`measure = [where ]`, \ + `dimension = `).", + ns.path, + malformed_measures, + malformed_dimensions + ); + } // `// macros` — this script is a workspace macro library: its body is // CREATE [OR REPLACE] MACRO statements plus plain setup, registered into // `macro_definition` and injected as TEMP macros into consumer jobs. @@ -1669,6 +1692,19 @@ async fn create_script_internal<'c>( } } + // Metric catalog: replace this path's declared measures/dimensions wholesale, + // so the catalog always describes the deployed state. Runs for every language, + // not just DuckDB: a script that drops its declarations (or changes language, + // or is replaced at the same path) must clear its old rows. + windmill_common::data_metrics::sync_metric_catalog( + &mut *tx, + &w_id, + &ns.path, + old_path, + &ns.content, + ) + .await?; + if ns.language == ScriptLang::DuckDb { // Record this script's macro-call edges for the asset graph (the // worker re-detects calls live at job time, so these are display @@ -4103,6 +4139,8 @@ async fn check_schema_contracts( &ann.column_lineage, &ann.data_tests, ann.materialize.as_ref(), + &ann.measures, + &ann.dimensions, ) .await?; tx.commit().await?; diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 9d37233228..8351ec0d2f 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -32,11 +32,9 @@ use windmill_common::DB; use ee_oss::validate_license_key; use windmill_common::usernames::generate_instance_username_for_all_users; -#[cfg(feature = "enterprise")] -use axum::extract::Query; use axum::{ body::Body, - extract::{Extension, Path}, + extract::{Extension, Path, Query}, response::Response, routing::{get, post}, Json, Router, @@ -283,15 +281,15 @@ pub async fn test_s3_bucket( let mut list = client.list(Some( &windmill_object_store::object_store_reexports::Path::from("".to_string()), )); - let first_file = list.next().await; - if first_file.is_some() { - if let Err(e) = first_file.as_ref().unwrap() { + match list.next().await { + Some(Err(e)) => { tracing::error!("error listing bucket: {e:#}"); - error::Error::internal_err(format!("Failed to list files in blob storage: {e:#}")); + return Err(error::Error::internal_err(format!( + "Failed to list files in blob storage: {e:#}" + ))); } - tracing::info!("Listed files: {:?}", first_file.unwrap()); - } else { - tracing::info!("No files in blob storage"); + Some(Ok(first_file)) => tracing::info!("Listed files: {:?}", first_file), + None => tracing::info!("No files in blob storage"), } let path = windmill_object_store::object_store_reexports::Path::from(format!( @@ -1580,6 +1578,7 @@ async fn refresh_custom_instance_user_pwd( ) -> JsonResult<()> { require_super_admin(&db, &authed.email).await?; windmill_common::utils::refresh_custom_instance_user_pwd(&db).await?; + windmill_common::utils::refresh_custom_instance_replication_user_pwd(&db).await?; Ok(Json(())) } @@ -1706,11 +1705,23 @@ async fn setup_custom_instance_pg_database_inner( )) })?; + // The replication attribute lives on a dedicated role used by postgres trigger + // connections. The getter creates the role (with its stored password) when the + // migration couldn't. + if let Err(e) = windmill_common::utils::get_custom_pg_instance_replication_password(db).await { + tracing::error!("Failed to ensure custom_instance_replication_user exists: {e:#}"); + } if let Err(e) = client - .batch_execute(&format!("ALTER ROLE custom_instance_user REPLICATION;")) + .batch_execute( + "ALTER ROLE custom_instance_replication_user REPLICATION; + GRANT custom_instance_user TO custom_instance_replication_user; + ALTER ROLE custom_instance_user NOREPLICATION;", + ) .await { - tracing::error!("Failed to grant replication permission to custom_instance_user: {e:#}"); + tracing::error!( + "Failed to grant replication permission to custom_instance_replication_user: {e:#}" + ); } logs.grant_permissions = "OK".to_string(); @@ -1972,25 +1983,53 @@ async fn fetch_resource_types_from_hub() -> error::Result, +} + async fn sync_cached_resource_types( Extension(db): Extension, authed: ApiAuthed, + Query(SyncResourceTypesQuery { name }): Query, ) -> error::Result { require_super_admin(&db, &authed.email).await?; use windmill_common::worker::HUB_RT_CACHE_DIR; let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); - let cached_types = match tokio::fs::read_to_string(&cache_path).await { - Ok(content) => serde_json::from_str::>(&content).map_err(|e| { - error::Error::InternalErr(format!("Failed to parse cached resource types: {}", e)) - })?, - Err(_) => fetch_resource_types_from_hub().await?, + // Manual sync is hub-first so it lands newly-published hub types on demand. The + // on-disk cache is only a fallback for when the hub is unreachable (airgapped + // installs / network error); refreshing it is left to the daily cache-rt cron and + // the startup sync in main.rs, which own the offline path. + let (resource_types, from_hub) = match fetch_resource_types_from_hub().await { + Ok(types) => { + tracing::info!("Fetched {} resource types live from the hub", types.len()); + (types, true) + } + Err(hub_err) => { + tracing::warn!( + "Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}" + ); + match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => { + let parsed = serde_json::from_str::>(&content) + .map_err(|e| { + error::Error::InternalErr(format!( + "Failed to parse cached resource types: {}", + e + )) + })?; + (parsed, false) + } + Err(_) => return Err(hub_err), + } + } }; let mut synced_count = 0; - for rt in &cached_types { + for rt in &resource_types { let exists: Option = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3)", &rt.name, @@ -2019,10 +2058,27 @@ async fn sync_cached_resource_types( synced_count += 1; } + // If a specific type was requested and is still absent after syncing, surface an + // explicit not-found instead of a silent "Synced 0". Word it by source so the + // cache-fallback path does not claim it checked the hub. + if let Some(name) = name.as_deref() { + if !resource_types.iter().any(|rt| rt.name == name) { + let source = if from_hub { + "on the hub" + } else { + "in the cached resource types (hub unreachable)" + }; + return Err(error::Error::NotFound(format!( + "resource type '{}' not found {}", + name, source + ))); + } + } + Ok(format!( "Synced {} resource types ({} unchanged)", synced_count, - cached_types.len() - synced_count + resource_types.len() - synced_count )) } diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ff0e8e754e..0e22f4a09d 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1751,6 +1751,7 @@ pub async fn delete_workspace_user_internal( "kafka_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "nats_trigger", "sqs_trigger", "gcp_trigger", @@ -1850,6 +1851,39 @@ pub async fn delete_workspace_user_internal( Ok(()) } +/// Non-admin path for `delete_workspace_user`: the creator of a fork may remove non-admin members +/// from the fork they created, so that adding the wrong collaborator is theirs to undo rather than +/// an admin's. Never on a root workspace, and never against an admin of the fork — the counterpart +/// of the add grant, whose bounds are spelled out on `add_user` in `windmill-api-workspaces`. +/// +/// `target_is_admin` must come from a row locked by the caller's deletion transaction: the grant +/// turns on the target not being an admin, so a promotion committing between the check and the +/// delete would remove an admin after all. `None` (no such member) is left to the caller's 404, +/// which is raised only after this returns so that a non-creator cannot probe who exists. +async fn authorize_fork_owner_delete_user( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + authed: &ApiAuthed, + username_to_delete: &str, + target_is_admin: Option, +) -> Result<()> { + if windmill_common::workspaces::fork_owned_by(&mut **tx, w_id, &authed.email) + .await? + .is_none() + { + return Err(Error::RequireAdmin(authed.username.clone())); + } + + if target_is_admin == Some(true) { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you cannot remove {username_to_delete}, who is an admin \ + of it" + ))); + } + + Ok(()) +} + async fn delete_workspace_user( authed: ApiAuthed, Extension(db): Extension, @@ -1857,17 +1891,27 @@ async fn delete_workspace_user( ) -> Result { let mut tx = db.begin().await?; - require_admin(authed.is_admin, &authed.username)?; - - let email_to_delete_o = sqlx::query_scalar!( - "SELECT email FROM usr where username = $1 AND workspace_id = $2", + // Locked so that the authorization below and the delete it guards see the same row. + let target = sqlx::query!( + "SELECT email, is_admin FROM usr where username = $1 AND workspace_id = $2 FOR UPDATE", username_to_delete, &w_id, ) .fetch_optional(&mut *tx) .await?; - let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; + if !authed.is_admin { + authorize_fork_owner_delete_user( + &mut tx, + &w_id, + &authed, + &username_to_delete, + target.as_ref().map(|t| t.is_admin), + ) + .await?; + } + + let email_to_delete = not_found_if_none(target, "User", &username_to_delete)?.email; delete_workspace_user_internal( &w_id, diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index d65316555e..f33e12470f 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -21,6 +21,7 @@ use windmill_common::{ jobs::{HIDE_WORKERS_FOR_NON_ADMINS, TAGS_ARE_SENSITIVE}, utils::{paginate, Pagination}, worker::{ALL_TAGS, CUSTOM_TAGS_PER_WORKSPACE, DEFAULT_TAGS, DEFAULT_TAGS_PER_WORKSPACE}, + workspaces::workspace_with_fork_ancestors, DB, }; @@ -161,9 +162,25 @@ async fn exists_workers_with_tags( let has_devops_role = require_devops_role(&db, &authed.email).await.is_ok(); if !has_devops_role { if let Some(ref workspace) = tags_query.workspace { + // This route is global, so the workspace is an unauthorized query param: check + // membership before reading its lineage, which would otherwise disclose whether + // an arbitrary workspace descends from one named by a `tag(parent*)` rule. + let is_member = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2 AND NOT disabled)", + workspace, + &authed.email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !is_member { + return Ok(Json(std::collections::HashMap::new())); + } + // Filter to only tags visible in this workspace + let chain = workspace_with_fork_ancestors(&db, workspace).await?; let custom_tags = CUSTOM_TAGS_PER_WORKSPACE.load(); - let allowed_tags = custom_tags.to_string_vec(Some(workspace.clone())); + let allowed_tags = custom_tags.to_string_vec(Some(&chain)); tags.retain(|t| allowed_tags.contains(t)); } else { // No workspace provided and not superadmin - return empty @@ -222,10 +239,12 @@ async fn get_custom_tags( async fn get_custom_tags_for_workspace( _authed: ApiAuthed, + Extension(db): Extension, Path(w_id): Path, ) -> JsonResult> { + let chain = workspace_with_fork_ancestors(&db, &w_id).await?; let tags_o = CUSTOM_TAGS_PER_WORKSPACE.load(); - let all_tags = tags_o.to_string_vec(Some(w_id)); + let all_tags = tags_o.to_string_vec(Some(&chain)); Ok(Json(all_tags)) } diff --git a/backend/windmill-api-workspaces/src/data_metrics.rs b/backend/windmill-api-workspaces/src/data_metrics.rs new file mode 100644 index 0000000000..c19df4a808 --- /dev/null +++ b/backend/windmill-api-workspaces/src/data_metrics.rs @@ -0,0 +1,204 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Read side of the metric catalog: what a DuckLake table declares (for the +//! script editor drawer) and what is declared under a path prefix (for agents). + +use axum::{ + extract::{Path, Query}, + routing::get, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use windmill_api_auth::{build_scope_path_filter, ApiAuthed, ScopePathFilter}; +use windmill_common::{ + data_metrics::{canonical_table_path, MetricEntry}, + db::UserDB, + error::{Error, JsonResult}, +}; + +pub fn workspaced_service() -> Router { + Router::new().route("/list", get(list_metrics)) +} + +#[derive(Deserialize)] +struct ListQuery { + /// `/
`, with or without the `ducklake://` scheme. + table: Option, + /// Path prefix to scope to, e.g. `f/analytics`. + path_prefix: Option, + /// Results per page, clamped to `MAX_PER_PAGE`. Defaults to `MAX_PER_PAGE`. + per_page: Option, + /// Keyset cursor: echo the previous response's `next_cursor` fields. All four + /// move together (they are the sort key); omit them for the first page. + cursor_table: Option, + cursor_kind: Option, + cursor_name: Option, + cursor_script: Option, +} + +/// Position on the (table_path, kind, name, script_path) sort key. Only ever built +/// from a row the caller received, so it never reveals a hidden row. +#[derive(Serialize)] +struct MetricCursor { + table_path: String, + kind: String, + name: String, + script_path: String, +} + +/// One page. `next_cursor` present means more rows may follow (pass it back); +/// absent means the catalog is exhausted. +#[derive(Serialize)] +struct MetricPage { + metrics: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + next_cursor: Option, +} + +/// Builds the descendant pattern for a path prefix, anchored to the `/` boundary +/// so `f/analytics` does not also match `f/analytics2`. LIKE wildcards in the +/// caller's own text are escaped, since an unescaped `%` or `_` would silently +/// widen the scope past the path they asked for. +fn descendants_of(prefix: &str) -> String { + let escaped = prefix + .replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_"); + format!("{escaped}/%") +} + +/// Largest page a caller may request; the caller pages with the keyset cursor. +const MAX_PER_PAGE: i64 = 1000; + +/// Lists declared measures and dimensions, optionally narrowed to one table or to +/// a producing script path prefix. +async fn list_metrics( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> JsonResult { + let table = q + .table + .as_deref() + .filter(|t| !t.is_empty()) + .map(canonical_table_path); + // A prefix matches the path itself and everything beneath it, but not a + // sibling folder that merely starts with the same characters. + let prefix = q + .path_prefix + .as_deref() + .map(|p| p.trim_end_matches('/')) + .filter(|p| !p.is_empty()) + .map(str::to_string); + let prefix_descendants = prefix.as_deref().map(descendants_of); + + // The four cursor components are one value; a partial cursor would compare + // against a tuple with a NULL (matching nothing) or silently restart. Reject it. + let cursor_present = [ + &q.cursor_table, + &q.cursor_kind, + &q.cursor_name, + &q.cursor_script, + ] + .iter() + .filter(|c| c.is_some()) + .count(); + if cursor_present != 0 && cursor_present != 4 { + return Err(Error::BadRequest( + "cursor_table, cursor_kind, cursor_name and cursor_script must be supplied together" + .to_string(), + )); + } + + // RLS reflects the user's own permissions but says nothing about a token's path + // scopes, so a scoped token must be filtered separately. This route has its own + // `data_metrics` scope domain (not an alias of `scripts`, which would let a + // metrics token reach every /scripts route). The grant is anchored on the + // producing script's path, as with `list_macros`. It is pushed INTO the query, + // not applied after the fetch: with scope filtering in SQL every returned row is + // authorized, so the keyset cursor (the last returned row) can never name a row + // the caller cannot see, and a post-fetch filter would instead let the page size + // reveal the count of out-of-scope declarations under a table/prefix. + let (scope_all, scope_exact, scope_prefix) = + match build_scope_path_filter(&authed, "data_metrics", "read") { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; + + let per_page = q.per_page.unwrap_or(MAX_PER_PAGE).clamp(1, MAX_PER_PAGE); + + let mut tx = user_db.begin(&authed).await?; + // Keyset paging: continue strictly after the previous page's last row on the + // (table_path, kind, name, script_path) sort key, so total work over the whole + // catalog is linear (offset paging re-reads every prior page). One extra row is + // fetched to learn whether more remain without a second query. + // + // The EXISTS runs under `script`'s RLS on this authed connection, so a caller + // only sees declarations from producers they can read. `data_metric` itself has + // no RLS, so removing this predicate would expose every workspace metric. + // Archived/deleted versions are excluded, else a renamed producer keeps serving + // its old path's declarations. The token scope filter ($10 allow-all, else the + // path is a $11 exact grant or sits at/under a $12 `prefix/*` grant on the `/` + // boundary) mirrors `ScopePathFilter::allows`. The ORDER BY is a total order + // (script_path breaks ties on table/kind/name), so keyset windows can't skip or + // duplicate a row. + let mut rows = sqlx::query_as!( + MetricEntry, + "SELECT script_path, table_path, kind, name, expr, filter \ + FROM data_metric dm \ + WHERE dm.workspace_id = $1 \ + AND ($2::text IS NULL OR dm.table_path = $2) \ + AND ($3::text IS NULL OR dm.script_path = $3 OR dm.script_path LIKE $4) \ + AND ($6::text IS NULL OR \ + (dm.table_path, dm.kind, dm.name, dm.script_path) > ($6, $7, $8, $9)) \ + AND ( $10 \ + OR dm.script_path = ANY($11) \ + OR EXISTS ( SELECT 1 FROM unnest($12::text[]) AS pfx \ + WHERE dm.script_path = pfx \ + OR left(dm.script_path, length(pfx) + 1) = pfx || '/' ) ) \ + AND EXISTS ( \ + SELECT 1 FROM script s \ + WHERE s.workspace_id = dm.workspace_id AND s.path = dm.script_path \ + AND s.archived = false AND s.deleted = false \ + ) \ + ORDER BY dm.table_path, dm.kind, dm.name, dm.script_path \ + LIMIT $5", + &w_id, + table.as_deref(), + prefix.as_deref(), + prefix_descendants.as_deref(), + per_page + 1, + q.cursor_table.as_deref(), + q.cursor_kind.as_deref(), + q.cursor_name.as_deref(), + q.cursor_script.as_deref(), + scope_all, + &scope_exact[..], + &scope_prefix[..], + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + // The probe row proves more remain; drop it and hand back its predecessor as the + // cursor. Without it the page fit entirely, so there is no next page. + let next_cursor = if rows.len() as i64 > per_page { + rows.truncate(per_page as usize); + rows.last().map(|last| MetricCursor { + table_path: last.table_path.clone(), + kind: last.kind.clone(), + name: last.name.clone(), + script_path: last.script_path.clone(), + }) + } else { + None + }; + Ok(Json(MetricPage { metrics: rows, next_cursor })) +} diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index c2b62d450b..017c9702a5 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -1,5 +1,6 @@ pub mod datatable_migrations; pub mod deployment_requests; +pub mod data_metrics; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index ba359cbffb..9089b6e8b6 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -135,6 +135,7 @@ pub fn workspaced_service() -> Router { .route("/edit_datatable_config", post(edit_datatable_config)) .merge(crate::datatable_migrations::routes()) .route("/git_sync_enabled", get(get_git_sync_enabled)) + .route("/git_sync_deploy_mode", get(get_git_sync_deploy_mode)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) .route( @@ -158,7 +159,6 @@ pub fn workspaced_service() -> Router { .route("/create_fork", post(create_workspace_fork)) .route("/attach_dev_workspace", post(attach_dev_workspace)) .route("/detach_dev_workspace", post(detach_dev_workspace)) - .route("/set_dev_workspace_label", post(set_dev_workspace_label)) .route("/get_dev_workspace", get(get_dev_workspace)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_color", post(change_workspace_color)) @@ -200,7 +200,7 @@ pub fn workspaced_service() -> Router { "/protection_rules/{rule_name}", post(update_protection_rule).delete(delete_protection_rule), ) - .route("/log_chat", post(log_ai_chat)) + .route("/log_feature_usage", post(log_feature_usage)) .route("/cloud_quotas", get(get_cloud_quotas)) .route("/prune_versions", post(prune_versions)) .route("/list_ws_specific", get(list_ws_specific)) @@ -302,6 +302,7 @@ pub struct WorkspaceSettings { pub success_handler: Option, #[serde(skip_serializing_if = "Option::is_none")] pub public_app_execution_limit_per_minute: Option, + pub error_handler_fallback_to_instance_alerts: bool, } /// Subset of `WorkspaceSettings` that is safe to return to any workspace @@ -451,6 +452,8 @@ struct CreateWorkspace { name: String, username: Option, color: Option, + #[serde(default)] + error_handler_fallback_to_instance_alerts: bool, } #[derive(Deserialize)] @@ -515,6 +518,10 @@ struct UserWorkspace { pub parent_workspace_id: Option, pub is_dev_workspace: bool, pub dev_workspace_label: Option, + /// Creator of the workspace (`workspace.owner`). On a fork it identifies the forker, who gets a + /// narrow membership grant over it even without being an admin — the UI keys the fork members + /// screen off this. + pub created_by: Option, pub disabled: bool, } @@ -554,6 +561,9 @@ pub struct EditErrorHandlerNew { pub muted_on_cancel: bool, #[serde(default)] pub muted_on_user_path: bool, + /// Left as `None` by clients that predate the setting (the CLI among them), which must + /// keep the stored value rather than silently reset it on every settings push. + pub fallback_to_instance_alerts: Option, } // Legacy format for error handler (flat fields from old CLI) @@ -582,6 +592,7 @@ impl EditErrorHandler { extra_args: legacy.error_handler_extra_args, muted_on_cancel: legacy.error_handler_muted_on_cancel, muted_on_user_path: false, // Old format doesn't have this field + fallback_to_instance_alerts: None, }, } } @@ -747,6 +758,184 @@ async fn list_workspaces( Ok(Json(workspaces)) } +/// Strip the server-only webhook HMAC secret from a `git_sync` blob before it is +/// returned to a client. The UI never needs it; it stays (encrypted) in the DB. +fn redact_git_sync_webhook_secrets(git_sync: &mut serde_json::Value) { + if let Some(repos) = git_sync + .get_mut("repositories") + .and_then(|r| r.as_array_mut()) + { + for repo in repos { + if let Some(auto_pull) = repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) { + auto_pull.remove("webhook_secret"); + } + } + } +} + +/// Zero the server-owned auto-pull fields (webhook id/secret/error, synced sha, +/// last pull status) on a client-supplied `AutoPullSettings`. The client only +/// controls `enabled` / `mode` / `poll_interval_s`; the rest is written by the +/// server (webhook creation, poller) and must never be trusted from the request — +/// otherwise a caller could inject a webhook id/secret or fake sync state. +fn clear_client_supplied_auto_pull_state( + auto_pull: &mut windmill_common::workspaces::AutoPullSettings, +) { + auto_pull.webhook_id = None; + auto_pull.webhook_secret = None; + auto_pull.webhook_error = None; + auto_pull.last_synced_sha = std::collections::HashMap::new(); + auto_pull.last_pull_status = None; +} + +/// A dev workspace deploys to a branch named after its environment label. If a +/// git-sync repository's tracked branch carries that same name, dev deploys +/// would write straight into the branch the workspace (or its prod) syncs +/// from — the CLI refuses that push, so every deploy job would fail. Reject +/// the label up front instead. +async fn reject_dev_label_matching_tracked_branch( + db: &DB, + label: Option<&str>, + workspace_ids: &[&str], +) -> Result<()> { + let label_branch = windmill_common::workspaces::dev_workspace_branch(label); + for w_id in workspace_ids { + let Some(settings) = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() + .and_then(|v| serde_json::from_value::(v).ok()) else { + continue; + }; + for repo in &settings.repositories { + let path = repo.git_repo_resource_path.trim_start_matches("$res:"); + let branch: Option = sqlx::query_scalar!( + "SELECT value->>'branch' FROM resource WHERE workspace_id = $1 AND path = $2", + w_id, + path + ) + .fetch_optional(db) + .await? + .flatten(); + if branch.as_deref() == Some(label_branch.as_str()) { + return Err(Error::BadRequest(format!( + "The environment label '{label_branch}' matches the tracked branch of git-sync \ + repository '{path}' in workspace '{w_id}': deploys from the dev workspace go \ + to the '{label_branch}' branch and would overwrite the branch that repository \ + syncs from. Use the other label or change the repository's tracked branch." + ))); + } + } + } + Ok(()) +} + +/// Reject parent-only git-sync settings on a fork workspace. Auto-pull and fork +/// PRs are configured at the parent: repo → fork sync is routed by the parent's +/// webhook/poller (`sync_forks`), and a fork-owned auto-pull would register a +/// second webhook on the same GitHub repo per fork. Promotion mode is rejected +/// on throwaway forks (their deploys always go to their `wm-fork/**` branch, so +/// a promotion repo could never take effect) but allowed on a **dev workspace**, +/// which deploys per-item `wm_deploy/**` branches that promote into the parent. +async fn reject_parent_only_git_sync_settings_on_fork<'a>( + db: &DB, + w_id: &str, + repos: impl Iterator, +) -> Result<()> { + let row = sqlx::query!( + "SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1", + w_id + ) + .fetch_optional(db) + .await?; + let is_fork = row + .as_ref() + .and_then(|r| r.parent_workspace_id.as_ref()) + .is_some() + || w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX); + if !is_fork { + return Ok(()); + } + let is_dev = row.map(|r| r.is_dev_workspace).unwrap_or(false); + let offending = repos.into_iter().find_map(|r| { + if r.auto_pull.as_ref().is_some_and(|a| a.enabled) { + Some("Auto-pull") + } else if r.use_individual_branch.unwrap_or(false) && !is_dev { + Some("Promotion mode") + } else if r.fork_open_prs { + Some("Opening PRs for fork deploys") + } else { + None + } + }); + if let Some(offending) = offending { + return Err(Error::BadRequest(format!( + "{offending} cannot be configured on a fork workspace: it is managed from the parent workspace's git sync settings" + ))); + } + Ok(()) +} + +/// Persist only the reconciled webhook fields (id/secret/error/mode) for `changed` +/// repos, one targeted JSONB update per repo (same pattern as the EE auto-pull +/// status writer). The webhook reconcile runs after the main save has committed, +/// so a read-modify-write of the whole blob would clobber a poller status write +/// or another settings save landing in the gap. `mode` is carried too: the +/// reconcile normalizes webhook -> polling for repos that can't register hooks, +/// and losing that would leave the poller skipping a webhook-mode repo that has +/// no webhook. A repo whose `auto_pull` was concurrently removed is left alone. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn persist_reconciled_webhook_fields( + db: &DB, + w_id: &str, + changed: &[(String, windmill_common::workspaces::AutoPullSettings)], +) -> Result<()> { + for (path, new_ap) in changed { + let mut patch = serde_json::Map::new(); + patch.insert( + "mode".to_string(), + serde_json::to_value(&new_ap.mode).map_err(|e| Error::internal_err(e.to_string()))?, + ); + if let Some(id) = new_ap.webhook_id { + patch.insert("webhook_id".to_string(), serde_json::json!(id)); + } + if let Some(secret) = &new_ap.webhook_secret { + patch.insert("webhook_secret".to_string(), serde_json::json!(secret)); + } + if let Some(err) = &new_ap.webhook_error { + patch.insert("webhook_error".to_string(), serde_json::json!(err)); + } + let patch = serde_json::Value::Object(patch); + sqlx::query!( + r#" + UPDATE workspace_settings + SET git_sync = jsonb_set( + git_sync, + '{repositories}', + (SELECT jsonb_agg( + CASE WHEN elem->>'git_repo_resource_path' = $2 + AND jsonb_typeof(elem->'auto_pull') = 'object' + THEN jsonb_set(elem, '{auto_pull}', + ((elem->'auto_pull') - 'webhook_id' - 'webhook_secret' - 'webhook_error') || $3) + ELSE elem END) + FROM jsonb_array_elements(git_sync->'repositories') AS elem) + ) + WHERE workspace_id = $1 + AND jsonb_typeof(git_sync->'repositories') = 'array' + "#, + w_id, + path, + patch, + ) + .execute(db) + .await?; + } + Ok(()) +} + async fn get_settings( authed: ApiAuthed, Path(w_id): Path, @@ -791,7 +980,8 @@ async fn get_settings( auto_invite, error_handler, success_handler, - public_app_execution_limit_per_minute + public_app_execution_limit_per_minute, + error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE @@ -803,9 +993,13 @@ async fn get_settings( .await .map_err(|e| Error::internal_err(format!("getting settings: {e:#}")))?; - let settings = not_found_if_none(settings, "workspace settings", &w_id)?; + let mut settings = not_found_if_none(settings, "workspace settings", &w_id)?; tx.commit().await?; + if let Some(git_sync) = settings.git_sync.as_mut() { + redact_git_sync_webhook_secrets(git_sync); + } + Ok(Json(settings)) } @@ -846,6 +1040,260 @@ async fn get_public_settings( Ok(Json(settings)) } +#[derive(Deserialize)] +pub struct GitSyncDeployModeQuery { + /// The branch the caller would push. + pub branch: Option, +} + +#[derive(Serialize, Debug)] +pub struct GitSyncDeployMode { + /// At least one git-sync repository is configured for this workspace. + pub configured: bool, + /// Pushing `branch` deploys via server-side auto-pull: exactly one licensed, + /// deliverable auto-pull repository tracks it. False (deploy via `git push` + /// through CI, or `wmill sync push`) when unlicensed, no repo tracks the + /// branch, or several do — with multiple synced repos we can't tell which the + /// local checkout is, so the caller asks the user instead. + pub deploy_on_push: bool, +} + +/// Whether an enabled auto-pull repo actually has a delivery path that fires, so +/// a push really deploys — mirroring the poller/webhook. Polling needs an HTTP(S) +/// URL (SSH is rejected in the background); webhook-only mode needs an active +/// hook; `auto` needs either. With neither, `enabled` alone never deploys (e.g. a +/// webhook that failed to register). +fn has_runnable_delivery( + auto_pull: &windmill_common::workspaces::AutoPullSettings, + resource: &serde_json::Value, +) -> bool { + let webhook_active = auto_pull.webhook_id.is_some(); + let is_app = resource + .get("is_github_app") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let is_http_url = resource + .get("url") + .and_then(|v| v.as_str()) + .map(|u| { + let u = u.trim_start(); + u.starts_with("https://") || u.starts_with("http://") + }) + .unwrap_or(false); + // App repos also have a GitHub-API poll fallback, but require an active + // webhook here — conservative (errs toward `wmill sync push`) rather than + // asserting the app installation can mint a token. + let can_poll = !is_app && is_http_url; + match auto_pull.mode { + windmill_common::workspaces::AutoPullMode::Webhook => webhook_active, + windmill_common::workspaces::AutoPullMode::Polling => can_poll, + windmill_common::workspaces::AutoPullMode::Auto => webhook_active || can_poll, + } +} + +/// Whether pushing `pushed_branch` matches a repo directly tracking +/// `tracked_branch` (the non-fork case). A blank tracked branch (repo default) is +/// unresolvable without a network call, so it never matches and the caller falls +/// back to `wmill sync push`. Fork/dev routing goes through +/// `windmill_common::workspaces::resolve_fork_branch_target` instead. +fn deploys_on_push_branch(pushed_branch: &str, tracked_branch: &str) -> bool { + !tracked_branch.is_empty() && pushed_branch == tracked_branch +} + +/// Non-admin endpoint so the CLI/agent can pick the deploy path (git push vs +/// `wmill sync push`) without reading the admin-only workspace settings. Takes +/// only the branch and returns booleans — no repository URLs, credentials, or +/// webhook config ever leave the backend. +async fn get_git_sync_deploy_mode( + _authed: ApiAuthed, + Path(w_id): Path, + Query(q): Query, + Extension(db): Extension, +) -> JsonResult { + // A fork clears its own auto-pull; its pushes deploy through the root + // ancestor's repo (which owns `sync_forks`), so evaluate the root's settings. + let ancestors = windmill_common::workspaces::fork_ancestor_chain(&db, &w_id).await?; + let is_fork = !ancestors.is_empty(); + let root_id = ancestors.last().cloned().unwrap_or_else(|| w_id.clone()); + + // Polling and webhook delivery both exclude deleted roots, so an archived + // root (or anything beneath one) can't deploy on push — treat a missing row + // as archived too. + let root_deleted = sqlx::query_scalar!("SELECT deleted FROM workspace WHERE id = $1", &root_id) + .fetch_optional(&db) + .await? + .unwrap_or(true); + + // Read on the plain pool: a fork member may not be a member of the root + // workspace, and only derived booleans are returned (never the settings). + let git_sync = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &root_id + ) + .fetch_optional(&db) + .await + .map_err(|e| Error::internal_err(format!("getting git_sync settings: {e:#}")))?; + + let settings = git_sync.flatten().and_then(|v| { + serde_json::from_value::(v) + .map_err(|e| { + tracing::warn!( + "git_sync deploy mode: settings deserialize failed for {root_id}: {e}" + ) + }) + .ok() + }); + + // Missing settings row / null git_sync means nothing is configured, not a 404. + let Some(settings) = settings else { + return Ok(Json(GitSyncDeployMode { + configured: false, + deploy_on_push: false, + })); + }; + + let configured = !settings.repositories.is_empty(); + + // Auto-pull runs only on Enterprise-licensed instances (see poll_git_auto_pull); + // without a caller branch there is nothing to match. Either way deploy_on_push + // stays false and the caller falls back (git push via CI, or wmill sync push). + let Some(branch) = q.branch.as_deref() else { + return Ok(Json(GitSyncDeployMode { + configured, + deploy_on_push: false, + })); + }; + let licensed = matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ); + + // Count the auto-pull repos that would deploy this branch. We deliberately do + // not check the caller's remote URL: with exactly one such repo the local + // checkout is unambiguously it, and with several we can't tell which is the + // caller's, so we report false and let the CLI ask the user. + let mut matches = 0u32; + if licensed && !root_deleted { + for repo in &settings.repositories { + let Some(auto_pull) = repo.auto_pull.as_ref() else { + continue; + }; + if !auto_pull.enabled { + continue; + } + // A fork deploys only through the root's sync_forks repos. + if is_fork && !auto_pull.sync_forks { + continue; + } + // Interpolates `$var:`/`$res:` as the auto-pull poller does. + // allow_cache=false: an on-demand status must reflect the current + // repo config, not a value cached by an earlier poll. + let Some(value) = windmill_store::resources::resolve_git_repository_resource( + &db, + &root_id, + &repo.git_repo_resource_path, + false, + ) + .await? + else { + continue; + }; + // `enabled` isn't enough: without a runnable delivery path (active + // webhook, or pollable non-app HTTPS repo) the push never deploys. + if !has_runnable_delivery(auto_pull, &value) { + continue; + } + let tracked_branch = value.get("branch").and_then(|v| v.as_str()).unwrap_or(""); + let deploys = if is_fork { + // Fork/dev routing (wm-fork/* or an env-label branch) resolved by + // the same logic the auto-pull reconciler uses; this repo counts + // only if the branch routes to *this* workspace. + windmill_common::workspaces::resolve_fork_branch_target( + &db, + &root_id, + &repo.git_repo_resource_path, + branch, + tracked_branch, + ) + .await? + .is_some_and(|(fork_id, _)| fork_id == w_id) + } else { + deploys_on_push_branch(branch, tracked_branch) + }; + if deploys { + matches += 1; + } + } + } + + Ok(Json(GitSyncDeployMode { + configured, + deploy_on_push: matches == 1, + })) +} + +#[cfg(test)] +mod git_sync_deploy_mode_tests { + use super::{deploys_on_push_branch, has_runnable_delivery}; + use serde_json::json; + use windmill_common::workspaces::{AutoPullMode, AutoPullSettings}; + + fn auto_pull(mode: AutoPullMode, webhook_id: Option) -> AutoPullSettings { + AutoPullSettings { enabled: true, mode, webhook_id, ..Default::default() } + } + + #[test] + fn runnable_delivery_requires_a_firing_path() { + let https = json!({ "url": "https://github.com/o/r.git" }); + let ssh = json!({ "url": "git@github.com:o/r.git" }); + let app = json!({ "url": "https://github.com/o/r.git", "is_github_app": true }); + // Polling serves only non-app HTTPS repos (SSH is rejected in background). + assert!(has_runnable_delivery( + &auto_pull(AutoPullMode::Auto, None), + &https + )); + assert!(has_runnable_delivery( + &auto_pull(AutoPullMode::Polling, None), + &https + )); + assert!(!has_runnable_delivery( + &auto_pull(AutoPullMode::Polling, None), + &ssh + )); + assert!(!has_runnable_delivery( + &auto_pull(AutoPullMode::Auto, None), + &ssh + )); + // Webhook-only mode needs an active hook. + assert!(!has_runnable_delivery( + &auto_pull(AutoPullMode::Webhook, None), + &https + )); + assert!(has_runnable_delivery( + &auto_pull(AutoPullMode::Webhook, Some(1)), + &https + )); + // App repos are gated on an active webhook here (conservative): their + // API poll-fallback may still deploy, so this is a safe under-report. + assert!(!has_runnable_delivery( + &auto_pull(AutoPullMode::Auto, None), + &app + )); + assert!(has_runnable_delivery( + &auto_pull(AutoPullMode::Auto, Some(1)), + &app + )); + } + + #[test] + fn non_fork_matches_tracked_branch_only() { + assert!(deploys_on_push_branch("main", "main")); + assert!(!deploys_on_push_branch("dev", "main")); + // An unresolved default (blank) tracked branch never matches. + assert!(!deploys_on_push_branch("main", "")); + } +} + async fn get_copilot_settings_state( _authed: ApiAuthed, Path(w_id): Path, @@ -2662,6 +3110,129 @@ fn cleanup_legacy_git_sync_settings_in_memory( #[cfg(not(feature = "enterprise"))] const CE_GIT_SYNC_MAX_USERS: i64 = 2; +/// Auto-pull is licensed per plan, not just per build: the poller only serves +/// Enterprise plans at runtime, so the save path must reject the setting too — +/// otherwise an EE binary without the plan could still register a webhook and +/// receive webhook-driven pulls. +#[cfg(feature = "enterprise")] +async fn check_git_sync_ee_license(feature: &str) -> Result<()> { + if !matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) { + return Err(Error::BadRequest(format!( + "{feature} requires an Enterprise license" + ))); + } + Ok(()) +} + +#[cfg(feature = "enterprise")] +async fn check_auto_pull_license() -> Result<()> { + check_git_sync_ee_license("Automatic pull from git").await +} + +/// In-app PR creation (promotion/fork deploy branches) drives GitHub API calls +/// from the deploy completion hook; runtime-gate it like auto-pull. +#[cfg(feature = "enterprise")] +async fn check_open_prs_license<'a>( + mut repos: impl Iterator, +) -> Result<()> { + if repos.any(|r| r.promotion_open_prs || r.fork_open_prs) { + check_git_sync_ee_license("Opening pull requests from Windmill").await?; + } + Ok(()) +} + +/// Promotion mode (`use_individual_branch`: per-item `wm_deploy/**` deploy +/// branches) is an EE feature; runtime-gate it like auto-pull and PR creation +/// so an enterprise binary without an active plan can't enable it via either +/// git-sync edit endpoint. +#[cfg(feature = "enterprise")] +async fn check_promotion_license<'a>( + mut repos: impl Iterator, +) -> Result<()> { + if repos.any(|r| r.use_individual_branch.unwrap_or(false)) { + check_git_sync_ee_license("Promotion mode").await?; + } + Ok(()) +} + +/// Promotion on a dev workspace needs the dev-aware sync script (hub >= 28796): +/// an older pinned script bundles a CLI that force-disables per-item branches +/// on every fork, so enabling promotion would silently keep deploying to the +/// env-label branch. Reject with an actionable error instead (the dispatcher +/// demotes inherited configs the same way). Roots run promotion on any script +/// version, and auto-managed repositories (no pin) always use the latest. +#[cfg(feature = "enterprise")] +async fn check_dev_promotion_script_version<'a>( + db: &DB, + w_id: &str, + repos: impl Iterator, +) -> Result<()> { + let mut offending: Option = None; + for r in repos { + if !r.use_individual_branch.unwrap_or(false) { + continue; + } + if !r.is_script_meets_min_version(28796)? { + offending = Some(r.effective_script_path().to_string()); + break; + } + } + let Some(offending) = offending else { + return Ok(()); + }; + let is_dev = sqlx::query!( + "SELECT parent_workspace_id, is_dev_workspace FROM workspace WHERE id = $1", + w_id + ) + .fetch_optional(db) + .await? + .map(|r| r.is_dev_workspace) + .unwrap_or(false); + if !is_dev { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Promotion mode on a dev workspace requires git sync script version 28796 or newer, \ + but this repository pins '{offending}'. Update the pinned sync script (or reset it to \ + auto-managed) first." + ))) +} + +/// A dev workspace's promotion must target its parent ("prod") workspace's own +/// git repository (same URL and branch) — that is what "promote to prod" means. +/// A fork-created dev inherits prod's repo; an **attached** dev keeps its own, +/// which may be unrelated. Reject enabling promotion on a repo the parent does +/// not track so the UI can't present an unrelated repo as prod's target. The +/// deploy path re-checks the same invariant (a resource edit could break it +/// after save), via the shared `dev_promotion_target_matches_parent`. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn check_dev_promotion_targets_parent_repo<'a>( + db: &DB, + w_id: &str, + repos: impl Iterator, +) -> Result<()> { + for r in repos.filter(|r| r.use_individual_branch.unwrap_or(false)) { + if !windmill_common::git_sync_ee::dev_promotion_target_matches_parent( + db, + w_id, + &r.git_repo_resource_path, + ) + .await? + { + return Err(Error::BadRequest( + "Promotion mode on a dev workspace must reuse the parent workspace's git repository \ + (same URL and branch), but this repository is not one the parent tracks — promotion \ + would target a repository the parent does not sync with." + .to_string(), + )); + } + } + Ok(()) +} + #[cfg(feature = "enterprise")] async fn check_git_sync_access(_db: &DB, _w_id: &str) -> Result<()> { Ok(()) @@ -2762,11 +3333,132 @@ async fn edit_git_sync_config( ) .await?; + // The whole-config save only writes the DB below; the managed GitHub webhooks + // are reconciled after the commit is durable (like the per-repository endpoint): + // `post_commit` carries the saved repos to reconcile + the hooks of repos this + // save removed, to delete. + #[cfg(all(feature = "enterprise", feature = "private"))] + let post_commit: Option<(WorkspaceGitSyncSettings, Vec<(String, i64)>)>; + if let Some(mut git_sync_settings) = new_config.git_sync_settings { + // Client-supplied server-owned auto-pull state is never trusted: strip it up + // front, then existing repos re-derive it from `existing` below and new repos + // stay clean. + for repo in git_sync_settings.repositories.iter_mut() { + if let Some(ap) = repo.auto_pull.as_mut() { + clear_client_supplied_auto_pull_state(ap); + } + repo.open_pr_error = None; + } + reject_parent_only_git_sync_settings_on_fork( + &db, + &w_id, + git_sync_settings.repositories.iter(), + ) + .await?; + // Auto-pull is EE-only (see edit_git_sync_repository). + #[cfg(not(feature = "enterprise"))] + if git_sync_settings + .repositories + .iter() + .any(|r| r.auto_pull.as_ref().is_some_and(|a| a.enabled)) + { + return Err(Error::BadRequest( + "Automatic pull from git is an Enterprise Edition feature".to_string(), + )); + } + #[cfg(feature = "enterprise")] + if git_sync_settings + .repositories + .iter() + .any(|r| r.auto_pull.as_ref().is_some_and(|a| a.enabled)) + { + check_auto_pull_license().await?; + } + #[cfg(feature = "enterprise")] + check_open_prs_license(git_sync_settings.repositories.iter()).await?; + #[cfg(feature = "enterprise")] + check_promotion_license(git_sync_settings.repositories.iter()).await?; + #[cfg(feature = "enterprise")] + check_dev_promotion_script_version(&db, &w_id, git_sync_settings.repositories.iter()) + .await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + check_dev_promotion_targets_parent_repo(&db, &w_id, git_sync_settings.repositories.iter()) + .await?; + // Promotion mode: EE only (mirrors edit_git_sync_repository). + #[cfg(not(feature = "enterprise"))] + if git_sync_settings + .repositories + .iter() + .any(|r| r.use_individual_branch.unwrap_or(false)) + { + return Err(Error::BadRequest( + "Promotion mode is an Enterprise Edition feature".to_string(), + )); + } + // Preserve server-owned auto-pull state (webhook id/secret, synced sha, last + // status) that the redacted GET response omits — otherwise a whole-config + // save from the UI would drop the webhook secret (breaking delivery) or + // clobber what the poller/webhook layer wrote. + let existing: Option = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|v| serde_json::from_value(v).ok()); + // Repos present before but absent from this save: their webhooks won't be + // reconciled below (no longer listed), so capture them for deletion. + #[cfg(all(feature = "enterprise", feature = "private"))] + let removed_webhooks: Vec<(String, i64)> = existing + .as_ref() + .map(|e| { + e.repositories + .iter() + .filter_map(|old| { + let hook = old.auto_pull.as_ref().and_then(|a| a.webhook_id)?; + // The save carries the hook forward (reconciled below) only + // when the repo is still present AND still has auto_pull — the + // preservation loop copies webhook fields only onto a Some + // auto_pull. Otherwise (repo dropped, or auto_pull cleared) the + // hook would orphan, so delete it. + let carried = git_sync_settings + .repositories + .iter() + .find(|n| n.git_repo_resource_path == old.git_repo_resource_path) + .map(|n| n.auto_pull.is_some()) + .unwrap_or(false); + (!carried).then_some((old.git_repo_resource_path.clone(), hook)) + }) + .collect() + }) + .unwrap_or_default(); + if let Some(existing) = &existing { + for repo in git_sync_settings.repositories.iter_mut() { + let Some(old) = existing + .repositories + .iter() + .find(|r| r.git_repo_resource_path == repo.git_repo_resource_path) + else { + continue; + }; + repo.open_pr_error = old.open_pr_error.clone(); + if let (Some(new_ap), Some(old_ap)) = + (repo.auto_pull.as_mut(), old.auto_pull.as_ref()) + { + new_ap.webhook_id = old_ap.webhook_id; + new_ap.webhook_secret = old_ap.webhook_secret.clone(); + new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); + new_ap.last_pull_status = old_ap.last_pull_status.clone(); + } + } + } + // Clean up legacy workspace-level settings if all repos are migrated cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); - let serialized_config = serde_json::to_value::(git_sync_settings) + let serialized_config = serde_json::to_value(&git_sync_settings) .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( @@ -2776,7 +3468,37 @@ async fn edit_git_sync_config( ) .execute(&mut *tx) .await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + { + post_commit = Some((git_sync_settings, removed_webhooks)); + } } else { + // Clearing the whole config removes every repo — delete all their webhooks. + #[cfg(all(feature = "enterprise", feature = "private"))] + { + let existing: Option = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|v| serde_json::from_value(v).ok()); + let removed_webhooks: Vec<(String, i64)> = existing + .map(|e| { + e.repositories + .iter() + .filter_map(|old| { + old.auto_pull + .as_ref() + .and_then(|a| a.webhook_id) + .map(|h| (old.git_repo_resource_path.clone(), h)) + }) + .collect() + }) + .unwrap_or_default(); + post_commit = Some((WorkspaceGitSyncSettings::default(), removed_webhooks)); + } sqlx::query!( "UPDATE workspace_settings SET git_sync = NULL WHERE workspace_id = $1", &w_id, @@ -2787,6 +3509,38 @@ async fn edit_git_sync_config( tx.commit().await?; + // Post-commit: reconcile each saved repo's managed webhook to match the config + // and delete the webhooks of repos this save removed. Best-effort — a failure + // leaves polling on. + #[cfg(all(feature = "enterprise", feature = "private"))] + if let Some((mut settings, removed_webhooks)) = post_commit { + let mut changed: Vec<(String, windmill_common::workspaces::AutoPullSettings)> = Vec::new(); + for repo in settings.repositories.iter_mut() { + let before = serde_json::to_value(&repo.auto_pull).ok(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await + { + tracing::warn!("git auto-pull: webhook sync error: {}", e); + } + if serde_json::to_value(&repo.auto_pull).ok() != before { + if let Some(ap) = repo.auto_pull.as_ref() { + changed.push((repo.git_repo_resource_path.clone(), ap.clone())); + } + } + } + if let Err(e) = persist_reconciled_webhook_fields(&db, &w_id, &changed).await { + tracing::warn!("git auto-pull: webhook field persist error: {}", e); + } + for (path, hook_id) in removed_webhooks { + if let Ok(url) = + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &w_id, &path).await + { + let _ = + windmill_common::git_sync_ee::delete_repo_webhook(&db, &w_id, &url, hook_id) + .await; + } + } + } + // Trigger git sync for git sync settings changes handle_deployment_metadata( &authed.email, @@ -2808,7 +3562,7 @@ async fn edit_git_sync_repository( Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(new_config): Json, + Json(mut new_config): Json, ) -> Result { require_admin(is_admin, &username)?; check_git_sync_access(&db, &w_id).await?; @@ -2816,6 +3570,53 @@ async fn edit_git_sync_repository( // Validate the resource path format validate_git_repo_resource_path(&new_config.git_repo_resource_path)?; + // Server-owned auto-pull state (webhook id/secret + sync status) is never + // accepted from the client — the webhook layer and poller own it. Strip it so an + // existing repo re-derives it from the DB (carried over below) and a new one + // starts clean. + if let Some(ap) = new_config.repository.auto_pull.as_mut() { + clear_client_supplied_auto_pull_state(ap); + } + new_config.repository.open_pr_error = None; + reject_parent_only_git_sync_settings_on_fork( + &db, + &w_id, + std::iter::once(&new_config.repository), + ) + .await?; + + // Auto-pull is EE-only: CE builds compile neither the poller nor the webhook + // reconciler, so accepting the setting would silently do nothing. + #[cfg(not(feature = "enterprise"))] + if new_config + .repository + .auto_pull + .as_ref() + .is_some_and(|a| a.enabled) + { + return Err(Error::BadRequest( + "Automatic pull from git is an Enterprise Edition feature".to_string(), + )); + } + #[cfg(feature = "enterprise")] + if new_config + .repository + .auto_pull + .as_ref() + .is_some_and(|a| a.enabled) + { + check_auto_pull_license().await?; + } + #[cfg(feature = "enterprise")] + check_open_prs_license(std::iter::once(&new_config.repository)).await?; + #[cfg(feature = "enterprise")] + check_promotion_license(std::iter::once(&new_config.repository)).await?; + #[cfg(feature = "enterprise")] + check_dev_promotion_script_version(&db, &w_id, std::iter::once(&new_config.repository)).await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + check_dev_promotion_targets_parent_repo(&db, &w_id, std::iter::once(&new_config.repository)) + .await?; + // Promotion mode: EE only #[cfg(not(feature = "enterprise"))] if new_config.repository.use_individual_branch.unwrap_or(false) { @@ -2893,8 +3694,32 @@ async fn edit_git_sync_repository( .find(|repo| repo.git_repo_resource_path == new_config.git_repo_resource_path); if let Some(existing_repo) = repo_found { - // Update existing repository - *existing_repo = new_config.repository; + // Update existing repository, but preserve server-owned auto-pull state + // (synced sha, last pull status, webhook id/secret) so a settings save + // from the UI cannot revert what the poller/webhook layer wrote. + let mut updated = new_config.repository; + updated.open_pr_error = existing_repo.open_pr_error.clone(); + match (updated.auto_pull.as_mut(), existing_repo.auto_pull.as_ref()) { + (Some(new_ap), Some(old_ap)) => { + new_ap.last_synced_sha = old_ap.last_synced_sha.clone(); + new_ap.last_pull_status = old_ap.last_pull_status.clone(); + new_ap.webhook_id = old_ap.webhook_id; + new_ap.webhook_secret = old_ap.webhook_secret.clone(); + } + // UI omitted auto_pull (e.g. older client): keep existing config. + (None, Some(_)) => { + updated.auto_pull = existing_repo.auto_pull.clone(); + } + _ => {} + } + // The request-side license gate above only saw the submitted config; the + // preservation can resurrect an enabled auto_pull (None arm), so re-check + // the effective state before it gets written and reconciled. + #[cfg(feature = "enterprise")] + if updated.auto_pull.as_ref().is_some_and(|a| a.enabled) { + check_auto_pull_license().await?; + } + *existing_repo = updated; } else { // Repository doesn't exist, add it as a new repository git_sync_settings.repositories.push(new_config.repository); @@ -2903,8 +3728,13 @@ async fn edit_git_sync_repository( // Clean up legacy workspace-level settings if all repos are migrated cleanup_legacy_git_sync_settings_in_memory(&mut git_sync_settings, &w_id); - // Save the updated configuration - let serialized_config = serde_json::to_value::(git_sync_settings) + // Save the updated configuration first, then reconcile the GitHub webhook to + // match it *after* the commit is durable (phase 2). Reconciling before the + // commit could leave the DB pointing at a hook that no longer matches if the + // save then failed (e.g. a delete on disable); post-commit reconciliation + // cannot. The pre-edit webhook id/secret are carried over above, so the + // committed row stays consistent until the reconcile persists any change. + let serialized_config = serde_json::to_value(&git_sync_settings) .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( @@ -2914,9 +3744,34 @@ async fn edit_git_sync_repository( ) .execute(&mut *tx) .await?; - tx.commit().await?; + // Post-commit: create/remove the webhook to match the saved config and persist + // the resulting hook id/secret. Best-effort — a failure leaves polling on. + #[cfg(all(feature = "enterprise", feature = "private"))] + { + let mut changed: Vec<(String, windmill_common::workspaces::AutoPullSettings)> = Vec::new(); + if let Some(repo) = git_sync_settings + .repositories + .iter_mut() + .find(|r| r.git_repo_resource_path == new_config.git_repo_resource_path) + { + let before = serde_json::to_value(&repo.auto_pull).ok(); + if let Err(e) = windmill_common::git_sync_ee::sync_repo_webhook(&db, &w_id, repo).await + { + tracing::warn!("git auto-pull: webhook sync error: {}", e); + } + if serde_json::to_value(&repo.auto_pull).ok() != before { + if let Some(ap) = repo.auto_pull.as_ref() { + changed.push((repo.git_repo_resource_path.clone(), ap.clone())); + } + } + } + if let Err(e) = persist_reconciled_webhook_fields(&db, &w_id, &changed).await { + tracing::warn!("git auto-pull: webhook field persist error: {}", e); + } + } + // Trigger git sync for individual repository update/add handle_deployment_metadata( &authed.email, @@ -2979,6 +3834,18 @@ async fn delete_git_sync_repository( WorkspaceGitSyncSettings::default() }; + // Capture the repo's managed webhook id; the hook itself is deleted only after + // the DB removal commits (below), so a failed save can't leave the repo pointing + // at a hook that no longer exists. Deletion bypasses the sync_repo_webhook + // lifecycle, so GitHub would otherwise keep delivering to an orphaned hook. + #[cfg(all(feature = "enterprise", feature = "private"))] + let webhook_to_delete: Option = git_sync_settings + .repositories + .iter() + .find(|r| r.git_repo_resource_path == request.git_repo_resource_path) + .and_then(|r| r.auto_pull.as_ref()) + .and_then(|a| a.webhook_id); + // Check if repository exists and remove it let original_count = git_sync_settings.repositories.len(); git_sync_settings @@ -3021,6 +3888,21 @@ async fn delete_git_sync_repository( tx.commit().await?; + // Removal is durable now — best-effort delete the GitHub webhook. + #[cfg(all(feature = "enterprise", feature = "private"))] + if let Some(hook_id) = webhook_to_delete { + if let Ok(url) = windmill_common::git_sync_ee::resolve_repo_url_interpolated( + &db, + &w_id, + &request.git_repo_resource_path, + ) + .await + { + let _ = + windmill_common::git_sync_ee::delete_repo_webhook(&db, &w_id, &url, hook_id).await; + } + } + // Trigger git sync for repository deletion handle_deployment_metadata( &authed.email, @@ -3308,6 +4190,19 @@ async fn edit_error_handler( let mut tx = db.begin().await?; + if let Some(fallback_to_instance_alerts) = ee.fallback_to_instance_alerts { + if fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &w_id).await?; + } + sqlx::query!( + "UPDATE workspace_settings SET error_handler_fallback_to_instance_alerts = $1 WHERE workspace_id = $2", + fallback_to_instance_alerts, + &w_id + ) + .execute(&mut *tx) + .await?; + } + sqlx::query_as!( Group, "INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING", @@ -3386,7 +4281,16 @@ async fn edit_error_handler( ActionKind::Update, &w_id, Some(&authed.email), - Some([("error_handler", &format!("{:?}", ee.path)[..])].into()), + Some( + [ + ("error_handler", &format!("{:?}", ee.path)[..]), + ( + "fallback_to_instance_alerts", + &format!("{:?}", ee.fallback_to_instance_alerts)[..], + ), + ] + .into(), + ), ) .await?; tx.commit().await?; @@ -3692,6 +4596,7 @@ struct UsedTriggers { pub nats_used: bool, pub postgres_used: bool, pub mqtt_used: bool, + pub amqp_used: bool, pub sqs_used: bool, pub gcp_used: bool, pub azure_used: bool, @@ -3717,6 +4622,7 @@ async fn get_used_triggers( EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!", EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!", EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS "mqtt_used!", + EXISTS(SELECT 1 FROM amqp_trigger WHERE workspace_id = $1) AS "amqp_used!", EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS "sqs_used!", EXISTS(SELECT 1 FROM gcp_trigger WHERE workspace_id = $1) AS "gcp_used!", EXISTS(SELECT 1 FROM azure_trigger WHERE workspace_id = $1) AS "azure_used!", @@ -3805,6 +4711,7 @@ async fn user_workspaces( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, workspace.is_dev_workspace, workspace.dev_workspace_label, + workspace.owner AS \"created_by?\", CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, usr.disabled FROM workspace @@ -3860,6 +4767,36 @@ async fn session_workspace_status( Ok(Json(statuses)) } +/// The instance critical alert channels belong to the instance operator, who on cloud is +/// not the workspace owner and never opted into a tenant's job failures. Fork workspaces run +/// throwaway copies of their parent's runnables, so instance-wide operational alerting must +/// stay a property of the real workspace. +async fn ensure_instance_alert_fallback_allowed<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, +) -> Result<()> { + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels is not available on cloud" + .to_string(), + )); + } + let is_fork = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#, + w_id + ) + .fetch_optional(&mut **tx) + .await? + .unwrap_or(false); + if is_fork { + return Err(Error::BadRequest( + "Reporting to the instance critical alert channels cannot be enabled on a fork workspace" + .to_string(), + )); + } + Ok(()) +} + pub async fn check_w_id_conflict<'c>(tx: &mut Transaction<'c, Postgres>, w_id: &str) -> Result<()> { if w_id == "global" { return Err(windmill_common::error::Error::BadRequest( @@ -4036,12 +4973,16 @@ async fn create_workspace( ) .execute(&mut *tx) .await?; + if nw.error_handler_fallback_to_instance_alerts { + ensure_instance_alert_fallback_allowed(&mut tx, &nw.id).await?; + } sqlx::query!( "INSERT INTO workspace_settings - (workspace_id, color) - VALUES ($1, $2)", + (workspace_id, color, error_handler_fallback_to_instance_alerts) + VALUES ($1, $2, $3)", nw.id, nw.color, + nw.error_handler_fallback_to_instance_alerts, ) .execute(&mut *tx) .await?; @@ -4189,6 +5130,7 @@ async fn clone_workspace_data( // Clone CI test references clone_ci_test_references(tx, source_workspace_id, target_workspace_id).await?; clone_macro_registry(tx, source_workspace_id, target_workspace_id).await?; + clone_metric_catalog(tx, source_workspace_id, target_workspace_id).await?; clone_asset_usages_and_triggers(tx, source_workspace_id, target_workspace_id).await?; // Clone flows with new versions @@ -4392,6 +5334,23 @@ async fn clone_triggers_and_schedules( .execute(&mut **tx) .await?; + sqlx::query!( + r#"INSERT INTO amqp_trigger ( + amqp_resource_path, queue_name, exchange, options, path, script_path, is_flow, + workspace_id, edited_by, edited_at, extra_perms, server_id, last_server_ping, + error, error_handler_path, error_handler_args, retry, mode, permissioned_as, labels + ) + SELECT + amqp_resource_path, queue_name, exchange, options, path, script_path, is_flow, + $1, edited_by, edited_at, extra_perms, NULL, NULL, + NULL, error_handler_path, error_handler_args, retry, 'disabled'::TRIGGER_MODE, permissioned_as, labels + FROM amqp_trigger WHERE workspace_id = $2"#, + target_workspace_id, + source_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( r#"INSERT INTO sqs_trigger ( path, queue_url, aws_resource_path, message_attributes, script_path, @@ -4540,6 +5499,18 @@ async fn update_workspace_settings( .into_iter() .filter(|r| !r.use_individual_branch.unwrap_or(false)) .take(1) + .map(|mut r| { + // Auto-pull and fork PRs are parent-owned and must not be inherited: + // the fork would otherwise carry the parent's webhook id (turning off + // auto-pull on the fork would delete the parent's webhook). A fork + // still inherits the push-direction config and the installation. + // Repo → fork sync is driven by the parent's webhook/poller + // (`sync_forks`), which routes the fork's `wm-fork/**` branch into it. + r.auto_pull = None; + r.fork_open_prs = false; + r.open_pr_error = None; + r + }) .collect(); let serialized_config = serde_json::to_value::(git_sync_settings) @@ -4832,6 +5803,26 @@ async fn clone_macro_registry( Ok(()) } +// Declared measures/dimensions are deploy-derived like the macro registry: +// without cloning them the fork's editor and agent tools report no metrics until +// every producer is manually redeployed there. +async fn clone_metric_catalog( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO data_metric (workspace_id, script_path, table_path, kind, name, expr, filter) + SELECT $2, script_path, table_path, kind, name, expr, filter + FROM data_metric WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + // Asset usage rows and `// on` subscriber triggers are deploy-derived like // ci_test_reference / the macro registry: without cloning them the fork's // pipeline graph has no asset nodes or lineage edges, and — worse — the asset @@ -5351,7 +6342,8 @@ async fn create_workspace_fork_branch( if nw.is_dev_workspace { validate_dev_workspace_id(&nw.id)?; // Reject a bad cosmetic label before any git branch is created (acted on in create_workspace_fork). - normalize_dev_workspace_label(nw.dev_workspace_label.clone())?; + let label = normalize_dev_workspace_label(nw.dev_workspace_label.clone())?; + reject_dev_label_matching_tracked_branch(&db, label.as_deref(), &[&w_id]).await?; ensure_dev_parent_is_root(&db, &w_id).await?; // Reject before creating any git branch if the parent already has a dev workspace, // otherwise the deferred branch-creation job leaves a dangling branch on the synced repos. @@ -5567,7 +6559,7 @@ async fn enforce_fork_depth( /// True if `raw` (the text form of a `json` value) contains a genuine `\u0000` /// NUL escape: a `u0000` preceded by an ODD run of backslashes. Mirrors the -/// parity rule in `strip_null_chars` (windmill-api `apps.rs`) — an even run +/// parity rule in `windmill_common::utils::strip_json_nul` — an even run /// (`\\u0000`) is an escaped backslash then the literal text "u0000" (common in /// minified JS) and is jsonb-safe. A genuine NUL is exactly what the /// `json`→`jsonb` re-encode in `clone_apps` / `clone_flows` rejects with @@ -5673,7 +6665,10 @@ async fn create_workspace_fork( validate_workspace_name(&nw.name)?; // Cosmetic label only applies to dev workspaces; a non-dev fork stores NULL. let dev_workspace_label = if nw.is_dev_workspace { - normalize_dev_workspace_label(nw.dev_workspace_label.clone())? + let label = normalize_dev_workspace_label(nw.dev_workspace_label.clone())?; + reject_dev_label_matching_tracked_branch(&db, label.as_deref(), &[&parent_workspace_id]) + .await?; + label } else { None }; @@ -5983,6 +6978,14 @@ async fn attach_dev_workspace( // The id is interpolated into a `wm-fork//` branch name like any fork. validate_dev_workspace_id(&dev_w_id)?; let dev_workspace_label = normalize_dev_workspace_label(req.dev_workspace_label.clone())?; + // The attached workspace keeps its own sync repos and prod keeps its config; + // the label branch must not collide with either side's tracked branch. + reject_dev_label_matching_tracked_branch( + &db, + dev_workspace_label.as_deref(), + &[&prod_w_id, &dev_w_id], + ) + .await?; let dev = sqlx::query!( r#"SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1"#, @@ -6057,14 +7060,57 @@ async fn attach_dev_workspace( ) .execute(&mut *tx) .await?; + // Clearing the instance-alert opt-in here keeps the stored setting truthful for a workspace + // that becomes parent-managed: dispatch enforces the fork boundary on its own, but a lingering + // `true` would survive a later detach and would make the settings page submit a value the API + // rejects on a fork. sqlx::query!( - "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + "UPDATE workspace_settings SET deploy_to = $1, error_handler_fallback_to_instance_alerts = false WHERE workspace_id = $2", &prod_w_id, &dev_w_id ) .execute(&mut *tx) .await?; + // The attached workspace is now parent-managed like any fork: its own + // auto-pull (and webhook), fork PRs, and promotion repos must not stay + // live — they'd keep pulling/pushing against its pre-attach tracked + // branch. Mirror the fork-creation copy: keep sync repos only, strip the + // parent-only fields, and delete any managed webhook after commit. + #[allow(unused_mut)] + let mut stripped_webhooks: Vec<(String, i64)> = Vec::new(); + if let Some(git_sync) = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &dev_w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + { + if let Ok(mut settings) = serde_json::from_value::(git_sync) { + settings + .repositories + .retain(|r| !r.use_individual_branch.unwrap_or(false)); + for r in settings.repositories.iter_mut() { + if let Some(hook) = r.auto_pull.as_ref().and_then(|a| a.webhook_id) { + stripped_webhooks.push((r.git_repo_resource_path.clone(), hook)); + } + r.auto_pull = None; + r.fork_open_prs = false; + r.open_pr_error = None; + } + let serialized = + serde_json::to_value(&settings).map_err(|e| Error::internal_err(e.to_string()))?; + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized, + &dev_w_id + ) + .execute(&mut *tx) + .await?; + } + } + if req.lock_prod_deploy || req.lock_prod_forking { lock_prod_workspace( &mut tx, @@ -6090,6 +7136,18 @@ async fn attach_dev_workspace( // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping // so per-workspace job tags route to the prod family immediately rather than after the TTL. windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Best-effort: the hooks captured before the strip above are unreachable now + // (their auto_pull is gone), so remove them from GitHub. + #[cfg(all(feature = "enterprise", feature = "private"))] + for (path, hook_id) in stripped_webhooks { + if let Ok(url) = + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &dev_w_id, &path).await + { + let _ = + windmill_common::git_sync_ee::delete_repo_webhook(&db, &dev_w_id, &url, hook_id) + .await; + } + } // Drop the cached ancestor chains too — the workspace existed BEFORE the attach, so a // cached empty chain reads as "not a fork" and its ducklake jobs would write the shared // lake until the TTL. Descendants' chains also gained the new root. @@ -6119,53 +7177,9 @@ async fn attach_dev_workspace( )) } -#[derive(Deserialize)] -struct SetDevWorkspaceLabel { - #[serde(default)] - dev_workspace_label: Option, -} - -/// Change the cosmetic display label ('dev' | 'staging') of the current workspace, which must itself -/// be a dev workspace. Purely visual (badge text + wording); requires admin of the dev workspace. -async fn set_dev_workspace_label( - authed: ApiAuthed, - Extension(db): Extension, - Path(w_id): Path, - Json(req): Json, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - let label = normalize_dev_workspace_label(req.dev_workspace_label)?; - - let mut tx = db.begin().await?; - let updated = sqlx::query_scalar!( - "UPDATE workspace SET dev_workspace_label = $1 WHERE id = $2 AND is_dev_workspace RETURNING id", - label, - &w_id, - ) - .fetch_optional(&mut *tx) - .await?; - if updated.is_none() { - return Err(Error::BadRequest(format!( - "Workspace '{w_id}' is not a dev workspace" - ))); - } - - audit_log( - &mut *tx, - &authed, - "workspaces.set_dev_workspace_label", - ActionKind::Update, - &w_id, - label.as_deref(), - None, - ) - .await?; - tx.commit().await?; - Ok(format!("Updated dev workspace label for {w_id}")) -} - /// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the -/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork). +/// prod lock. Whether `parent_workspace_id` is kept depends on the workspace's origin (see the +/// UPDATE below): a genuine fork stays a fork, a standalone workspace returns to standalone. async fn detach_dev_workspace( authed: ApiAuthed, Extension(db): Extension, @@ -6194,8 +7208,14 @@ async fn detach_dev_workspace( } let mut tx = db.begin().await?; + // A wm-fork- workspace re-designated as dev returns to being a plain fork + // (keeps its parent); a standalone workspace that was attached returns to + // being standalone — with the parent kept it would still classify as a + // fork and deploy to wm-fork/** branches forever. sqlx::query!( - "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + "UPDATE workspace SET is_dev_workspace = false, + parent_workspace_id = CASE WHEN id LIKE 'wm-fork-%' THEN parent_workspace_id ELSE NULL END + WHERE id = $1", &dev_w_id ) .execute(&mut *tx) @@ -6222,6 +7242,20 @@ async fn detach_dev_workspace( tx.commit().await?; windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); + // The parent link may just have been cleared (standalone workspace that was + // attached): drop the caches that resolved it, mirroring attach. + windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_billing_workspace_cache(&id); + } + } Ok(format!( "Detached dev workspace {} from {}", @@ -6632,6 +7666,72 @@ If you do not have an account on {}, login with SSO or ask an admin to create an )) } +/// Non-admin path for `add_user`: the creator of a fork may bring collaborators into the fork they +/// created, so a team can work on it without an admin of the fork having to step in. The grant is +/// deliberately narrow, because a fork holds a full clone of its parent (secrets included) and the +/// creator may be an ordinary developer: +/// - only on a fork they created, never on a root workspace; +/// - only as a developer, so it can never mint an admin (nor an operator, which would need the +/// workspace's operator settings to be meaningful); +/// - only for someone who is already a developer or admin of the parent, so pulling them into the +/// fork cannot widen who can read the parent's data. +/// +/// Anything outside those bounds stays an admin's call. Returns the username the new member must be +/// given in the fork. +async fn authorize_fork_owner_add_user( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + nu: &NewWorkspaceUser, +) -> Result { + let parent = windmill_common::workspaces::fork_owned_by(db, w_id, &authed.email) + .await? + .ok_or_else(|| Error::RequireAdmin(authed.username.clone()))?; + + if nu.is_admin || nu.operator { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you can only add members as developers; ask an admin of \ + {w_id} for any other role" + ))); + } + + let parent_username = sqlx::query_scalar!( + "SELECT username FROM usr + WHERE workspace_id = $1 AND email = $2 AND NOT operator AND NOT disabled", + parent, + nu.email, + ) + .fetch_optional(db) + .await?; + + let Some(parent_username) = parent_username else { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you can only add developers or admins of its parent \ + workspace {parent}; {} is not one, so only an admin of {w_id} can add them", + nu.email + ))); + }; + + // Ownership of a `u//` path is decided by the username alone, and the fork holds a + // clone of every such path from the parent. Seating the new member on a username other than + // their own would therefore hand them that parent user's cloned scripts, variables and secrets + // — so their parent username is the only one they may be given here, whatever the caller asked + // for (`add_user` otherwise lets the caller choose it when AUTOMATE_USERNAME_CREATION is off). + if nu + .username + .as_deref() + .is_some_and(|u| !u.is_empty() && u != parent_username) + { + return Err(Error::PermissionDenied(format!( + "as the creator of fork {w_id} you cannot choose the username of a member you add; {} \ + joins as '{parent_username}', the username they already have in {parent}", + nu.email + ))); + } + + Ok(parent_username) +} + async fn add_user( authed: ApiAuthed, Extension(db): Extension, @@ -6639,8 +7739,6 @@ async fn add_user( Path(w_id): Path, Json(mut nu): Json, ) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - #[cfg(not(feature = "enterprise"))] if w_id == "admins" { return Err(Error::BadRequest( @@ -6650,6 +7748,12 @@ async fn add_user( nu.email = nu.email.to_lowercase(); + let fork_owner_username = if !authed.is_admin { + Some(authorize_fork_owner_add_user(&db, &w_id, &authed, &nu).await?) + } else { + None + }; + #[cfg(feature = "enterprise")] if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_new_user(&db, &nu.email, nu.operator).await? @@ -6685,7 +7789,9 @@ async fn add_user( .flatten() .unwrap_or(true); - let username = if automate_username_creation { + let username = if let Some(username) = fork_owner_username { + username + } else if automate_username_creation { if nu.username.is_some() && nu.username.unwrap().len() > 0 { return Err(Error::BadRequest( "username is not allowed when username creation is automated".to_string(), @@ -8940,25 +10046,98 @@ const TRIGGER_OR_SCHEDULE_TABLES: &[&str] = &[ "email_trigger", ]; +const MAX_FEATURE_USAGE_EVENTS: usize = 50; + #[derive(Deserialize)] -struct LogAiChatPayload { - session_id: String, - provider: String, - model: String, - mode: String, +struct FeatureUsageEvent { + feature: String, + kind: String, + #[serde(default)] + key: String, + #[serde(default)] + entity_id: String, + value: Option, } -async fn log_ai_chat( +#[derive(Deserialize)] +struct LogFeatureUsagePayload { + events: Vec, +} + +// Only registered (feature, kind) actions are accepted, so telemetry stays +// limited to predefined feature actions. Keys are shape-checked (identifier-like, +// no spaces) rather than pinned to value sets: they come from our own frontend +// (modes, tab/draft kinds, tool names, provider:model) and pinning every value +// server-side was not worth the maintenance. +const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[ + ("ai_session", "created"), + ("ai_session", "message"), + ("ai_session", "autonomy"), + ("ai_session", "tab"), + ("ai_session", "tokens"), + ("ai_session", "deployed"), + ("ai_session", "archived"), + ("ai_session", "deleted"), + ("ai_session", "beta_optout"), + ("ai_session", "beta_optin"), + ("ai_chat", "message"), + ("ai_chat", "model"), + ("ai_chat", "tool"), +]; + +fn is_identifier_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/')) +} + +fn valid_feature_usage_event(e: &FeatureUsageEvent) -> bool { + FEATURE_USAGE_KINDS.contains(&(e.feature.as_str(), e.kind.as_str())) + && (e.key.is_empty() || is_identifier_shaped(&e.key, 100)) + && (e.entity_id.is_empty() || is_identifier_shaped(&e.entity_id, 50)) +} + +async fn log_feature_usage( Extension(db): Extension, - Json(payload): Json, + Json(payload): Json, ) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a + // single INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String, String), i64> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_FEATURE_USAGE_EVENTS) { + if !valid_feature_usage_event(&e) { + continue; + } + let value = e.value.unwrap_or(1).clamp(1, 1_000_000); + *agg.entry((e.feature, e.kind, e.key, e.entity_id)) + .or_insert(0) += value; + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + let mut features = Vec::with_capacity(agg.len()); + let mut kinds = Vec::with_capacity(agg.len()); + let mut keys = Vec::with_capacity(agg.len()); + let mut entity_ids = Vec::with_capacity(agg.len()); + let mut values = Vec::with_capacity(agg.len()); + for ((feature, kind, key, entity_id), value) in agg { + features.push(feature); + kinds.push(kind); + keys.push(key); + entity_ids.push(entity_id); + values.push(value); + } sqlx::query!( - "INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4) - ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1", - &payload.session_id, - &payload.provider, - &payload.model, - &payload.mode + "INSERT INTO feature_usage (feature, kind, key, entity_id, value) + SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[]) + ON CONFLICT (feature, kind, key, entity_id, day) + DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + &features, + &kinds, + &keys, + &entity_ids, + &values ) .execute(&db) .await?; diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 6b4ace5f0a..7998a769f9 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -108,13 +108,60 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, deploy_to, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, ducklake, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) .execute(&mut *tx) .await?; + // The managed git-sync webhooks deliver to /api/w/{old_id}/... — a URL the + // renamed workspace no longer answers on (the old id is archived and the + // receiver skips it). Strip the webhook fields from the new row so polling + // resumes at the normal interval and the next settings save re-registers a + // hook with the new URL; the stale hooks are deleted after commit. + #[allow(unused_mut)] + let mut stale_webhooks: Vec<(String, i64)> = Vec::new(); + if let Some(git_sync) = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &rw.new_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + { + if let Ok(mut settings) = serde_json::from_value::< + windmill_common::workspaces::WorkspaceGitSyncSettings, + >(git_sync) + { + let mut changed = false; + for r in settings.repositories.iter_mut() { + if let Some(ap) = r.auto_pull.as_mut() { + if let Some(hook) = ap.webhook_id { + stale_webhooks.push((r.git_repo_resource_path.clone(), hook)); + } + changed |= ap.webhook_id.is_some() + || ap.webhook_secret.is_some() + || ap.webhook_error.is_some(); + ap.webhook_id = None; + ap.webhook_secret = None; + ap.webhook_error = None; + } + } + if changed { + let serialized = serde_json::to_value(&settings) + .map_err(|e| Error::internal_err(e.to_string()))?; + sqlx::query!( + "UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2", + serialized, + &rw.new_id + ) + .execute(&mut *tx) + .await?; + } + } + } + info!("Duplicating workspace_key table"); sqlx::query!( "INSERT INTO workspace_key SELECT $1, kind, key FROM workspace_key WHERE workspace_id = $2", @@ -232,6 +279,15 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + info!("Updating amqp_trigger table"); + sqlx::query!( + "UPDATE amqp_trigger SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + info!("Updating gcp_trigger table"); sqlx::query!( "UPDATE gcp_trigger SET workspace_id = $1 WHERE workspace_id = $2", @@ -744,6 +800,20 @@ pub(crate) async fn change_workspace_id( tx.commit().await?; + // Best-effort: the hooks stripped above still exist on GitHub pointing at + // the old workspace URL; remove them (resources already live under the new id). + #[cfg(all(feature = "enterprise", feature = "private"))] + for (path, hook_id) in stale_webhooks { + if let Ok(url) = + windmill_common::git_sync_ee::resolve_repo_url_interpolated(&db, &rw.new_id, &path) + .await + { + let _ = + windmill_common::git_sync_ee::delete_repo_webhook(&db, &rw.new_id, &url, hook_id) + .await; + } + } + // The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent // routing cache and their billing-workspace mapping so jobs route + meter under the renamed root // rather than the old (archived) one, instead of waiting for the caches' TTLs. Deeper descendants diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index fa9ed3f2e5..ee789945f5 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-amqp?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-amqp?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] @@ -26,7 +26,11 @@ kafka = ["dep:windmill-trigger-kafka", "windmill-store/kafka"] kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] nats = ["dep:windmill-trigger-nats", "windmill-store/nats"] websocket = ["dep:windmill-trigger-websocket"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email"] +smtp = ["instance_smtp", "dep:mail-parser", "dep:openssl", "dep:windmill-trigger-email"] +# Outbound instance-SMTP email (the send_email_with_instance_smtp endpoint and +# critical alerts) without the inbound email trigger's openssl/mail-parser deps. +# `smtp` is the full trigger + endpoint; `instance_smtp` is the endpoint only. +instance_smtp = ["windmill-common/smtp"] license = ["dep:rsa", "windmill-api-settings/license"] zip = ["dep:async_zip"] oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"] @@ -34,6 +38,7 @@ http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-store/http static_frontend = ["dep:rust-embed"] postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-store/postgres_trigger"] mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-store/mqtt_trigger"] +amqp_trigger = ["dep:windmill-trigger-amqp", "windmill-store/amqp_trigger"] native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "dep:strum", "oauth2"] sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-store/sqs_trigger"] gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-store/gcp_trigger"] @@ -141,6 +146,7 @@ matchit = { workspace = true, optional = true } windmill-trigger-kafka = { workspace = true, optional = true } windmill-trigger-postgres = { workspace = true, optional = true } windmill-trigger-mqtt = { workspace = true, optional = true } +windmill-trigger-amqp = { workspace = true, optional = true } windmill-trigger-websocket = { workspace = true, optional = true } windmill-trigger-email = { workspace = true, optional = true } windmill-trigger-nats = { workspace = true, optional = true } diff --git a/backend/windmill-api/docs_snapshot/llms-full.txt.gz b/backend/windmill-api/docs_snapshot/llms-full.txt.gz index cc52e79f2c..b453b899bb 100644 Binary files a/backend/windmill-api/docs_snapshot/llms-full.txt.gz and b/backend/windmill-api/docs_snapshot/llms-full.txt.gz differ diff --git a/backend/windmill-api/docs_snapshot/llms.txt.gz b/backend/windmill-api/docs_snapshot/llms.txt.gz index 111dc1537f..07db118b4c 100644 Binary files a/backend/windmill-api/docs_snapshot/llms.txt.gz and b/backend/windmill-api/docs_snapshot/llms.txt.gz differ diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index aa086cb02c..e2e83f1a4a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.759.0 + version: 1.770.0 title: Windmill API contact: @@ -847,7 +847,7 @@ paths: /w/{workspace}/users/delete/{username}: delete: - summary: delete user (require admin privilege) + summary: delete user (require admin privilege, except for the creator of a fork removing a non-admin member of the fork) operationId: deleteUser tags: - user @@ -1274,32 +1274,6 @@ paths: - id - name - /w/{workspace}/workspaces/set_dev_workspace_label: - post: - summary: set the cosmetic display label (dev/staging) of this dev workspace - operationId: setDevWorkspaceLabel - tags: - - workspace - parameters: - - $ref: "#/components/parameters/WorkspaceId" - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - dev_workspace_label: - type: string - enum: [dev, staging] - responses: - "200": - description: dev workspace label updated - content: - text/plain: - schema: - type: string - /workspaces/exists: post: summary: exists workspace @@ -1356,7 +1330,7 @@ paths: /settings/refresh_custom_instance_user_pwd: post: - summary: Refreshes the password for the custom_instance_user + summary: Refreshes the passwords for the custom_instance_user and the custom_instance_replication_user (used by postgres triggers) operationId: refreshCustomInstanceUserPwd tags: - setting @@ -3052,7 +3026,7 @@ paths: /w/{workspace}/workspaces/add_user: post: - summary: add user to workspace + summary: add user to workspace (require admin privilege, except for the creator of a fork adding a developer/admin of its parent workspace as a developer of the fork) operationId: addUser tags: - workspace @@ -3643,6 +3617,9 @@ paths: public_app_execution_limit_per_minute: type: integer description: Rate limit for public app executions per minute per server. NULL or 0 means disabled. + error_handler_fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. /w/{workspace}/workspaces/get_deploy_to: get: @@ -4491,6 +4468,94 @@ paths: items: type: string + /w/{workspace}/data_metrics/list: + get: + summary: list declared measures and dimensions on DuckLake tables + description: > + Call this before writing any aggregate query over a DuckLake table. A + declared measure is the canonical definition of that number, and + reproducing it yourself will silently disagree with it (a `revenue` + measure typically excludes refunds or test rows). Filter by `table` for + one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for + everything declared under a folder; omit both to browse the whole + catalog. Results are keyset-paged: a full page may mean more remain, so + continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a + `filter` write it as + `expr FILTER (WHERE filter)` so measures with different predicates can + share one GROUP BY. If a number you need has no declared measure, write + your own aggregate as usual. Results are limited to declarations whose + producing script the caller can read. + operationId: listDataMetrics + x-mcp-tool: true + tags: + - data_metric + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: table + in: query + description: DuckLake table path, with or without the `ducklake://` scheme + schema: + type: string + - name: path_prefix + in: query + description: Producing script path prefix, e.g. `f/analytics` + schema: + type: string + - name: per_page + in: query + description: Results per page, capped at 1000 (default 1000) + schema: + type: integer + - name: cursor_table + in: query + description: > + Keyset cursor. To page, pass the previous response's `next_cursor` fields + back as `cursor_*`; all four move together, and are omitted for the first + page. Continue whenever `next_cursor` is present. Every returned row is one + the caller may read, so the cursor never names a hidden row. + schema: + type: string + - name: cursor_kind + in: query + schema: + type: string + - name: cursor_name + in: query + schema: + type: string + - name: cursor_script + in: query + schema: + type: string + responses: + "200": + description: declared measures and dimensions + content: + application/json: + schema: + type: object + required: [metrics] + properties: + metrics: + type: array + items: + $ref: "#/components/schemas/DataMetric" + next_cursor: + description: > + Present when more rows may follow: pass its fields back as the + `cursor_*` params. Absent means the catalog is exhausted. + type: object + required: [table_path, kind, name, script_path] + properties: + table_path: + type: string + kind: + type: string + name: + type: string + script_path: + type: string + /w/{workspace}/workspaces/list_datatables: get: summary: list Datatables @@ -5209,6 +5274,45 @@ paths: type: integer nullable: true + /w/{workspace}/workspaces/git_sync_deploy_mode: + get: + summary: Get how local changes deploy to this workspace via git sync + operationId: getGitSyncDeployMode + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: branch + in: query + required: false + description: The branch the caller would push. + schema: + type: string + responses: + "200": + description: Git sync deploy mode + content: + application/json: + schema: + type: object + required: + - configured + - deploy_on_push + properties: + configured: + type: boolean + description: At least one git-sync repository is configured. + deploy_on_push: + type: boolean + description: >- + True means a `git push` is confirmed to deploy via + auto-pull: exactly one licensed, deliverable repository + tracks the branch. False is *not confirmed* rather than a + definite no — it also covers unlicensed, ambiguous + (several repos track it), and conservative false-negatives; + determine the deploy path another way (CI `git push`, or + `wmill sync push`). + /w/{workspace}/workspaces/edit_git_sync_config: post: summary: edit workspace git sync settings @@ -5528,6 +5632,8 @@ paths: type: boolean mqtt_used: type: boolean + amqp_used: + type: boolean gcp_used: type: boolean azure_used: @@ -5549,6 +5655,7 @@ paths: - nats_used - postgres_used - mqtt_used + - amqp_used - gcp_used - azure_used - sqs_used @@ -6531,10 +6638,10 @@ paths: "400": description: invalid input or request closed - /w/{workspace}/workspaces/log_chat: + /w/{workspace}/workspaces/log_feature_usage: post: - summary: log AI chat message - operationId: logAiChat + summary: log anonymous feature usage telemetry events + operationId: logFeatureUsage tags: - workspace parameters: @@ -6546,19 +6653,26 @@ paths: schema: type: object required: - - session_id - - provider - - model - - mode + - events properties: - session_id: - type: string - provider: - type: string - model: - type: string - mode: - type: string + events: + type: array + items: + type: object + required: + - feature + - kind + properties: + feature: + type: string + kind: + type: string + key: + type: string + entity_id: + type: string + value: + type: integer responses: "204": description: logged @@ -8598,6 +8712,11 @@ paths: description: List every draft in the workspace (all users), not just the current user's own + legacy rows. Other users' rows come back with `mine=false` (view-only). schema: type: boolean + - name: compare_to_workspace + in: query + description: A fork passes its parent workspace id here to have each row flagged with `unchanged_from_parent`. Ignored unless it is exactly this workspace's parent. + schema: + type: string responses: "200": description: the user's drafts @@ -8633,6 +8752,9 @@ paths: mine: type: boolean description: The row belongs to the current user (own draft or the legacy no-owner row) and is therefore actionable. Always true in the default listing; with `all_users=true`, other users' rows are false (view-only). + unchanged_from_parent: + type: boolean + description: Only present when `compare_to_workspace` was passed. True when this draft is identical to the parent's draft at the same path/kind/owner (cloned in on fork and never edited here). draft_users: description: | Draft authors at this (path, kind) — the legacy NULL-email row surfaced as a null username. @@ -10359,6 +10481,73 @@ paths: - path - value + /w/{workspace}/runnables/list: + get: + summary: list runnables (scripts, flows, apps) merged, ordered and keyset-paginated + operationId: listRunnables + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: order_by + in: query + description: "sort key: 'updated' (default) or 'name'" + schema: + type: string + enum: + - updated + - name + - $ref: "#/components/parameters/OrderDesc" + - name: kinds + in: query + description: comma-separated subset of script,flow,app (default all) + schema: + type: string + - name: show_archived + in: query + schema: + type: boolean + - name: include_without_main + in: query + description: include library scripts (no runnable main) + schema: + type: boolean + - name: path_start + in: query + description: restrict to paths under this prefix + schema: + type: string + - name: label + in: query + schema: + type: string + - name: search + in: query + description: case-insensitive substring match on summary or path + schema: + type: string + - $ref: "#/components/parameters/PerPage" + - name: cursor + in: query + description: opaque keyset cursor from a previous page's next_cursor + schema: + type: string + responses: + "200": + description: a page of merged, ordered runnables + content: + application/json: + schema: + type: object + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/RunnableItem" + next_cursor: + type: string /w/{workspace}/flows/list: get: summary: list all flows @@ -10793,7 +10982,7 @@ paths: application/json: schema: allOf: - - $ref: "#/components/schemas/OpenFlowWPath" + - $ref: "#/components/schemas/EditFlow" - type: object properties: deployment_message: @@ -12139,6 +12328,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: FileMetadata @@ -12191,6 +12382,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: FilePreview @@ -12241,6 +12434,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Parquet Preview @@ -12293,6 +12488,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Csv Preview @@ -12325,6 +12522,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: Table count @@ -12354,6 +12553,8 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/S3Sig" + - $ref: "#/components/parameters/S3Exp" responses: "200": description: The downloaded file @@ -14570,6 +14771,44 @@ paths: - resume - cancel + /w/{workspace}/jobs/wac_approval_urls/{id}/{step_key}: + get: + summary: get the resume urls bound to a specific wait_for_approval step of a workflow-as-code job + operationId: getWacApprovalUrls + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + - name: step_key + in: path + required: true + description: checkpoint key of the wait_for_approval step, as passed to `wait_for_approval(key=...)` + schema: + type: string + - name: approver + in: query + schema: + type: string + responses: + "200": + description: url endpoints + content: + application/json: + schema: + type: object + properties: + approvalPage: + type: string + resume: + type: string + cancel: + type: string + required: + - approvalPage + - resume + - cancel + /w/{workspace}/jobs/slack_approval/{id}: get: summary: generate interactive slack approval for suspended job @@ -17329,6 +17568,211 @@ paths: schema: type: string + /w/{workspace}/amqp_triggers/create: + post: + summary: create amqp trigger + operationId: createAmqpTrigger + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new amqp trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewAmqpTrigger" + responses: + "201": + description: amqp trigger created + content: + text/plain: + schema: + type: string + + /w/{workspace}/amqp_triggers/update/{path}: + post: + summary: update amqp trigger + operationId: updateAmqpTrigger + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EditAmqpTrigger" + responses: + "200": + description: amqp trigger updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/amqp_triggers/delete/{path}: + delete: + summary: delete amqp trigger + operationId: deleteAmqpTrigger + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: amqp trigger deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/amqp_triggers/get/{path}: + get: + summary: get amqp trigger + operationId: getAmqpTrigger + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/GetDraft" + responses: + "200": + description: amqp trigger retrieved + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/AmqpTrigger" + - $ref: "#/components/schemas/UserDraftOverlay" + + /w/{workspace}/amqp_triggers/list: + get: + summary: list amqp triggers + operationId: listAmqpTriggers + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + required: true + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + - name: label + in: query + required: false + schema: + type: string + description: Filter by label + - $ref: "#/components/parameters/IncludeDraftOnly" + responses: + "200": + description: amqp trigger list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AmqpTrigger" + + /w/{workspace}/amqp_triggers/exists/{path}: + get: + summary: does amqp trigger exists + operationId: existsAmqpTrigger + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: amqp trigger exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/amqp_triggers/setmode/{path}: + post: + summary: set enabled amqp trigger + operationId: setAmqpTriggerMode + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated amqp trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + mode: + $ref: "#/components/schemas/TriggerMode" + force: + type: boolean + description: > + Bypass the parent-state conflict warning when enabling a + trigger in a fork whose parent has the same path enabled. + required: + - mode + responses: + "200": + description: amqp trigger enabled set + content: + text/plain: + schema: + type: string + + /w/{workspace}/amqp_triggers/test: + post: + summary: test amqp connection + operationId: testAmqpConnection + tags: + - amqp_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: test amqp connection + required: true + content: + application/json: + schema: + type: object + properties: + amqp_resource_path: + type: string + description: Path to the AMQP resource containing broker connection configuration + required: + - amqp_resource_path + responses: + "200": + description: successfully connected to amqp + content: + text/plain: + schema: + type: string + /w/{workspace}/gcp_triggers/create: post: summary: create gcp trigger @@ -19025,6 +19469,8 @@ paths: - folder parameters: - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" - name: only_member_of in: query description: only list the folders the user is member of (default false) @@ -19838,6 +20284,7 @@ paths: nats_trigger, postgres_trigger, mqtt_trigger, + amqp_trigger, gcp_trigger, azure_trigger, sqs_trigger, @@ -19885,6 +20332,7 @@ paths: nats_trigger, postgres_trigger, mqtt_trigger, + amqp_trigger, gcp_trigger, azure_trigger, sqs_trigger, @@ -19943,6 +20391,7 @@ paths: nats_trigger, postgres_trigger, mqtt_trigger, + amqp_trigger, gcp_trigger, azure_trigger, sqs_trigger, @@ -20547,6 +20996,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, test the connection of this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: Connection settings @@ -20628,6 +21082,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, list the files of this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: List of file keys @@ -20664,6 +21123,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, load the file metadata from this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: FileMetadata @@ -20713,6 +21177,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, load the file preview from this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: FilePreview @@ -21023,6 +21492,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, delete the file from this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: Confirmation @@ -21052,6 +21526,11 @@ paths: in: query schema: type: string + - name: s3_resource_path + in: query + description: When set, move the file within this object storage resource instead of the workspace storage + schema: + type: string responses: "200": description: Confirmation @@ -22556,6 +23035,508 @@ paths: schema: type: string + /w/{workspace}/hub/publish_draft: + post: + summary: create or update a hub project draft + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubDraft + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishDraftBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts: + post: + summary: publish a script to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScript + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishScriptBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows: + post: + summary: publish a flow to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlow + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishFlowBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/apps: + post: + summary: publish an app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps: + post: + summary: publish a raw app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishRawAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps/{id}/embed: + post: + summary: set or clear the embed url of a hub raw app + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawAppEmbed + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: hub id of the raw app + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RawAppEmbedBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts/{ask_id}/recording: + post: + summary: attach a recording to a hub script + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScriptRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: ask_id + in: path + required: true + description: hub ask id of the script + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows/{flow_id}/recording: + post: + summary: attach a recording to a hub flow + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlowRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: flow_id + in: path + required: true + description: hub id of the flow + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/pipeline_recording: + post: + summary: attach a data-pipeline recording to a hub project + description: | + Requires the caller to be a workspace admin. A data-pipeline recording is + scoped to the whole project (a folder cascade), not a single item. Forwards + the request to the configured Hub scoped to the `{workspace}:{folder}` + source and returns the Hub's status code and raw response body. + operationId: publishHubPipelineRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug + schema: + $ref: "#/components/schemas/HubProjectSlug" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PipelineRecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resource_types: + post: + summary: publish a resource type to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResourceType + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourceTypeBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resources: + post: + summary: publish resource placeholders to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResources + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourcesBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/triggers: + post: + summary: publish triggers to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubTriggers + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishTriggersBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/migrations: + post: + summary: publish data table migrations to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubMigrations + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishMigrationsBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/export: + get: + summary: export a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub and returns the Hub's status code and raw response body. + The folder scope is only needed to re-export the caller's own draft; + approved projects are public, so it is optional here. + operationId: getHubProjectExport + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - name: folder + in: query + required: false + description: folder scoping the Hub project source (`{workspace}:{folder}`) + schema: + type: string + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/logo: + post: + summary: set or clear a hub project's custom logo + description: | + Requires the caller to be a workspace admin. Sets the project's custom + logo (base64 png/svg, decoded size max 512KB) or clears it when `logo` + is null; the `logo` field itself is required so an empty body cannot + clear the logo by accident. Forwards the request to the configured Hub + scoped to the `{workspace}:{folder}` source and returns the Hub's + status code and raw response body. + operationId: publishHubProjectLogo + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug + schema: + $ref: "#/components/schemas/HubProjectSlug" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectLogoBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/submit: + post: + summary: submit a hub project draft for review + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: submitHubProject + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/project: + get: + summary: get the hub project linked to a workspace folder + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: getHubProjectBySource + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + components: securitySchemes: bearerAuth: @@ -22604,6 +23585,16 @@ components: required: true schema: type: string + HubPublishFolder: + name: folder + in: query + required: true + description: | + workspace folder scoping the Hub publication: a workspace can publish + one Hub project per folder and the Hub-side source key is + `{workspace}:{folder}` + schema: + type: string PublicationName: name: publication in: path @@ -22660,6 +23651,18 @@ components: required: true schema: type: string + S3Sig: + name: sig + in: query + description: HMAC signature of a presigned S3 object (bypasses the app provenance gate) + schema: + type: string + S3Exp: + name: exp + in: query + description: Expiry timestamp of a presigned S3 object signature + schema: + type: string CustomPath: name: custom_path in: path @@ -23089,6 +24092,7 @@ components: - trigger_kafka - trigger_nats - trigger_mqtt + - trigger_amqp - trigger_sqs - trigger_gcp - trigger_azure @@ -23404,6 +24408,9 @@ components: muted_on_user_path: type: boolean default: false + fallback_to_instance_alerts: + type: boolean + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Omit to leave the stored value untouched. EditErrorHandlerLegacy: type: object @@ -23890,6 +24897,86 @@ components: - language - content + RunnableItem: + type: object + description: | + A row in the merged runnables listing. `type` is the discriminator; + kind-specific fields (hash/language/kind for scripts, execution_mode/ + version for apps) are present only for that kind. `edited_at` is the + unified last-updated time (a script's created_at, a flow/app's edit + time). + properties: + type: + type: string + enum: + - script + - flow + - app + path: + type: string + summary: + type: string + workspace_id: + type: string + extra_perms: + type: object + additionalProperties: + type: boolean + starred: + type: boolean + archived: + type: boolean + is_draft: + type: boolean + draft_only: + type: boolean + nullable: true + draft_path: + type: string + draft_users: + type: array + items: + type: object + labels: + type: array + items: + type: string + inherited_labels: + type: array + items: + type: string + ws_error_handler_muted: + type: boolean + edited_at: + type: string + format: date-time + hash: + type: string + description: script version hash as a 16-char hex string + language: + type: string + kind: + type: string + auto_kind: + type: string + use_codebase: + type: boolean + has_deploy_errors: + type: boolean + raw_app: + type: boolean + execution_mode: + type: string + id: + type: integer + format: int64 + version: + type: integer + format: int64 + required: + - type + - path + Script: type: object properties: @@ -26073,6 +27160,7 @@ components: - slack - teams - email + - instance_alerts NewSchedule: type: object @@ -26293,6 +27381,7 @@ components: - kafka - nats - mqtt + - amqp - sqs - gcp - azure @@ -26797,6 +27886,8 @@ components: type: number mqtt_count: type: number + amqp_count: + type: number gcp_count: type: number azure_count: @@ -27313,6 +28404,183 @@ components: - subscribe_topics - mqtt_resource_path + AmqpExchange: + type: object + properties: + exchange_name: + type: string + description: Name of the exchange to bind the consumed queue to + routing_keys: + type: array + items: + type: string + description: Routing keys used to bind the queue to the exchange + required: + - exchange_name + + AmqpOptions: + type: object + properties: + declare_queue: + type: boolean + description: Declare the queue (durable) before consuming; when false the queue is declared passively and must already exist + prefetch_count: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + description: Maximum number of unacknowledged messages the broker delivers at once (1-65535) + + AmqpTrigger: + allOf: + - $ref: "#/components/schemas/TriggerExtraProperty" + type: object + properties: + amqp_resource_path: + type: string + description: Path to the AMQP resource containing broker connection configuration + queue_name: + type: string + description: Name of the queue to consume messages from + exchange: + $ref: "#/components/schemas/AmqpExchange" + nullable: true + description: Optional exchange binding for the consumed queue + options: + $ref: "#/components/schemas/AmqpOptions" + nullable: true + description: Optional consumer options (queue declaration, prefetch) + server_id: + type: string + description: ID of the server currently handling this trigger (internal) + last_server_ping: + type: string + format: date-time + description: Timestamp of last server heartbeat (internal) + error: + type: string + description: Last error message if the trigger failed + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler + retry: + $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions + required: + - amqp_resource_path + - queue_name + + NewAmqpTrigger: + type: object + properties: + amqp_resource_path: + type: string + description: Path to the AMQP resource containing broker connection configuration + queue_name: + type: string + description: Name of the queue to consume messages from + exchange: + nullable: true + $ref: "#/components/schemas/AmqpExchange" + description: Optional exchange binding for the consumed queue + options: + nullable: true + $ref: "#/components/schemas/AmqpOptions" + description: Optional consumer options (queue declaration, prefetch) + path: + type: string + description: The unique Windmill path for this trigger. Must be of the form `u//` or `f//`. + script_path: + type: string + description: Path to the script or flow to execute when a message is received + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + mode: + $ref: "#/components/schemas/TriggerMode" + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler + retry: + $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions + permissioned_as: + type: string + description: The user or group this trigger runs as. Used during deployment to preserve the original trigger owner. + preserve_permissioned_as: + type: boolean + description: "When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it." + labels: + type: array + items: + type: string + required: + - path + - script_path + - is_flow + - amqp_resource_path + - queue_name + + EditAmqpTrigger: + type: object + properties: + amqp_resource_path: + type: string + description: Path to the AMQP resource containing broker connection configuration + queue_name: + type: string + description: Name of the queue to consume messages from + exchange: + nullable: true + $ref: "#/components/schemas/AmqpExchange" + description: Optional exchange binding for the consumed queue + options: + nullable: true + $ref: "#/components/schemas/AmqpOptions" + description: Optional consumer options (queue declaration, prefetch) + path: + type: string + description: The unique Windmill path for this trigger. Must be of the form `u//` or `f//`. + script_path: + type: string + description: Path to the script or flow to execute when a message is received + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + mode: + $ref: "#/components/schemas/TriggerMode" + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + $ref: "#/components/schemas/ScriptArgs" + description: Arguments to pass to the error handler + retry: + $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" + description: Retry configuration for failed executions + permissioned_as: + type: string + description: The user or group this trigger runs as. Used during deployment to preserve the original trigger owner. + preserve_permissioned_as: + type: boolean + description: "When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it." + labels: + type: array + items: + type: string + required: + - path + - script_path + - is_flow + - amqp_resource_path + - queue_name + DeliveryType: type: string enum: @@ -28821,6 +30089,10 @@ components: type: string color: type: string + error_handler_fallback_to_instance_alerts: + type: boolean + default: false + description: Report failed jobs to the instance critical alert channels when no workspace error handler is set. Not available on cloud or on fork workspaces. required: - id - name @@ -29109,6 +30381,37 @@ components: type: string required: - path + # Like OpenFlowWPath but `path` is optional: on update the flow is identified by + # the URL, so the body path is only needed to rename it. Kept as a separate schema + # (rather than making OpenFlowWPath.path optional) so createFlow still requires path. + EditFlow: + allOf: + - $ref: "../../openflow.openapi.yaml#/components/schemas/OpenFlow" + - type: object + properties: + path: + type: string + tag: + type: string + ws_error_handler_muted: + type: boolean + priority: + type: integer + dedicated_worker: + type: boolean + timeout: + type: number + visible_to_runner_only: + type: boolean + on_behalf_of_email: + type: string + preserve_on_behalf_of: + type: boolean + description: "When true and the caller is a member of the 'wm_deployers' group, preserves the original on_behalf_of_email value instead of overwriting it." + labels: + type: array + items: + type: string FlowPreview: type: object @@ -29927,9 +31230,71 @@ components: type: array items: $ref: "#/components/schemas/GitSyncObjectType" + auto_pull: + $ref: "#/components/schemas/AutoPullSettings" + promotion_open_prs: + type: boolean + fork_open_prs: + type: boolean + open_pr_error: + type: string + description: server-owned, last failure opening a PR for a deploy branch of this repo required: - git_repo_resource_path + AutoPullMode: + type: string + enum: + - auto + - webhook + - polling + + AutoPullStatus: + type: object + properties: + synced_sha: + type: string + at: + type: integer + format: int64 + job_id: + type: string + format: uuid + success: + type: boolean + error: + type: string + required: + - at + - success + + AutoPullSettings: + type: object + properties: + enabled: + type: boolean + mode: + $ref: "#/components/schemas/AutoPullMode" + poll_interval_s: + type: integer + sync_forks: + type: boolean + webhook_id: + type: integer + format: int64 + webhook_secret: + type: string + webhook_error: + type: string + last_synced_sha: + type: object + additionalProperties: + type: string + last_pull_status: + $ref: "#/components/schemas/AutoPullStatus" + required: + - enabled + MetricMetadata: type: object properties: @@ -30178,6 +31543,7 @@ components: postgres, sqs, mqtt, + amqp, gcp, azure, email, @@ -30360,6 +31726,7 @@ components: "nats_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", @@ -30692,6 +32059,33 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + DataMetric: + description: | + One `// measure` or `// dimension` declaration, as catalogued from the + script that materializes the table. `expr` and `filter` are the author's + own SQL: a reader renders a measure as `expr` plus, when `filter` is set, + a trailing `FILTER (WHERE filter)`. + type: object + required: [script_path, table_path, kind, name, expr] + properties: + script_path: + type: string + description: The declaring script, and the path reads are authorized against + table_path: + type: string + description: Canonical scheme-less DuckLake path, `/.
` (schema defaults to `main`) + kind: + type: string + enum: + - measure + - dimension + name: + type: string + expr: + type: string + filter: + type: string + description: Row predicate from a measure's trailing `where` ContractWarning: description: | One save-time schema-contract warning: a consumer reference that does @@ -30708,6 +32102,9 @@ components: - missing_lineage_source - missing_relationship_column - relationship_type_mismatch + - missing_measure_column + - missing_dimension_column + - non_aggregate_measure - suppressed asset_path: type: string @@ -31200,3 +32597,302 @@ components: - name - owner - private + + HubProjectSlug: + type: string + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + + PublishDraftBody: + type: object + properties: + slug: + $ref: "#/components/schemas/HubProjectSlug" + name: + type: string + summary: + type: string + readme: + type: string + required: + - slug + - name + - summary + + PublishScriptBody: + type: object + properties: + summary: + type: string + app: + type: string + description: + type: string + kind: + type: string + content: + type: string + language: + type: string + schema: + type: object + lockfile: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - summary + - app + - content + - language + - project_slug + + PublishFlowInner: + type: object + properties: + summary: + type: string + description: + type: string + value: + type: object + schema: + type: object + required: + - summary + - value + + PublishFlowBody: + type: object + properties: + flow: + $ref: "#/components/schemas/PublishFlowInner" + apps: + type: array + items: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - flow + - apps + - project_slug + + PublishAppBody: + type: object + properties: + app: + type: object + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - app + - apps + - summary + - project_slug + + PublishRawAppBody: + type: object + properties: + raw: + type: string + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - raw + - apps + - summary + - project_slug + + RawAppEmbedBody: + type: object + properties: + external_embed_url: + type: string + nullable: true + description: explicit `null` clears the embed (unpublish) + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + RecordingBody: + type: object + properties: + recording: + type: object + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + PipelineRecordingBody: + type: object + properties: + recording: + type: object + + ProjectLogoBody: + type: object + properties: + logo: + description: the logo to set, or null to clear the project's current logo + nullable: true + type: object + properties: + b64: + description: base64-encoded image bytes (decoded size max 512KB) + type: string + mime: + type: string + enum: [image/png, image/svg+xml] + required: + - b64 + - mime + required: + - logo + + PublishResourceTypeBody: + type: object + properties: + name: + type: string + schema: + type: object + description: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - name + - project_slug + + PublishResourceBody: + type: object + properties: + path: + type: string + resource_type: + type: string + required: + - path + - resource_type + + PublishResourcesBody: + type: object + properties: + resources: + type: array + items: + $ref: "#/components/schemas/PublishResourceBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - resources + - project_slug + + PublishTriggerBody: + type: object + properties: + path: + type: string + kind: + type: string + summary: + type: string + nullable: true + description: + type: string + nullable: true + config: + type: object + script_ask_id: + type: integer + format: int64 + nullable: true + flow_id: + type: integer + format: int64 + nullable: true + required: + - path + - kind + - config + + PublishTriggersBody: + type: object + properties: + triggers: + type: array + items: + $ref: "#/components/schemas/PublishTriggerBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - triggers + - project_slug + + PublishMigrationBody: + type: object + description: one best-effort data table migration attached to a project (per data table) + properties: + datatable_name: + type: string + sql: + type: string + sql_down: + type: string + description: defaults to an empty string when omitted + enabled: + type: boolean + required: + - datatable_name + - sql + - enabled + + PublishMigrationsBody: + type: object + properties: + migrations: + type: array + items: + $ref: "#/components/schemas/PublishMigrationBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - migrations + - project_slug diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index a92a9830f4..f722b9b565 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -104,18 +104,7 @@ lazy_static::lazy_static! { } }; - pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() - .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) - .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) - .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) - // The SSRF check in `get_base_url` only validates the configured `base_url`. - // reqwest follows up to 10 redirects by default and does not revalidate the - // hops, so a public base_url could 3xx the server into a private/internal - // address. Disable redirect following so the validated host is the only one - // we ever connect to. AI APIs respond directly and do not rely on redirects, - // so this holds even for ALLOW_PRIVATE_AI_BASE_URLS deployments. - .redirect(reqwest::redirect::Policy::none()) - .user_agent("windmill/beta")) + pub(crate) static ref HTTP_CLIENT: Client = ai_http_client_builder() .build() .expect("Failed to build AI HTTP client - check system TLS configuration"); @@ -129,6 +118,65 @@ pub(crate) fn invalidate_ai_request_cache_for_workspace(workspace_id: &str) { AI_REQUEST_CACHE.retain(|(cached_workspace_id, _), _| cached_workspace_id != workspace_id); } +/// Shared configuration for every outbound AI HTTP client (the pooled +/// [`HTTP_CLIENT`] and the per-request DNS-pinned clients). +/// +/// Redirects are disabled: the SSRF check only validates the configured host, so +/// a public host that 3xx-es could otherwise bounce us to a private/internal +/// address. AI APIs respond directly and do not rely on redirects, so this holds +/// even for ALLOW_PRIVATE_AI_BASE_URLS deployments. DNS pinning likewise only +/// covers the original host, so following a redirect would reopen the hole. +fn ai_http_client_builder() -> reqwest::ClientBuilder { + configure_client( + reqwest::ClientBuilder::new() + .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) + .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) + .pool_idle_timeout(Some(std::time::Duration::from_secs( + HTTP_POOL_IDLE_TIMEOUT_SECS, + ))) + .redirect(reqwest::redirect::Policy::none()) + .user_agent("windmill/beta"), + ) +} + +/// Build the client for a single outbound AI request to `url`, pinning DNS to +/// the SSRF-validated address so the connect cannot rebind to an internal IP +/// after the check (DNS-rebinding TOCTOU). +/// +/// Returns the shared pooled [`HTTP_CLIENT`] unchanged when there is nothing to +/// pin — an IP-literal host, or a deployment that opted into private AI +/// endpoints via `ALLOW_PRIVATE_AI_BASE_URLS`. The same opt-out and error hint +/// as `get_base_url` apply, so the guard here is consistent with save-time +/// validation while additionally closing the connect-time window. +async fn pinned_ai_client_for(url: &str) -> Result> { + use std::borrow::Cow; + use windmill_common::ssrf::SsrfValidationError; + + if *windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { + return Ok(Cow::Borrowed(&HTTP_CLIENT)); + } + + let target = windmill_common::ssrf::validate_url_for_ssrf(url) + .await + .map_err(|e| match e { + e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( + "{e}. If you need to use private/internal AI endpoints, \ + set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" + )), + e => Error::from(e), + })?; + + if target.pinned_addrs().is_empty() { + return Ok(Cow::Borrowed(&HTTP_CLIENT)); + } + + let client = target + .apply_dns_pinning(ai_http_client_builder()) + .build() + .map_err(to_anyhow)?; + Ok(Cow::Owned(client)) +} + #[derive(Deserialize, Debug)] struct AIOAuthResource { client_id: String, @@ -303,23 +351,13 @@ async fn get_token_using_oauth( // Validate the resolved token_url against SSRF rules before issuing the request, // mirroring the protection applied to base_url in `get_base_url` (same // ALLOW_PRIVATE_AI_BASE_URLS opt-in). Without this a workspace member could - // point token_url at an internal/metadata address. - if !*windmill_ai::ai_providers::ALLOW_PRIVATE_AI_BASE_URLS { - use windmill_common::ssrf::SsrfValidationError; - windmill_common::ssrf::validate_url_for_ssrf(&resource.token_url) - .await - .map_err(|e| match e { - e @ SsrfValidationError::Private { .. } => Error::BadRequest(format!( - "{e}. If you need to use private/internal AI endpoints, \ - set the ALLOW_PRIVATE_AI_BASE_URLS=true environment variable" - )), - e => Error::from(e), - })?; - } + // point token_url at an internal/metadata address. The returned client pins + // DNS to the validated address so the connect cannot rebind after the check. + let client = pinned_ai_client_for(&resource.token_url).await?; let mut params = HashMap::new(); params.insert("grant_type", "client_credentials"); params.insert("scope", "https://cognitiveservices.azure.com/.default"); - let response = HTTP_CLIENT + let response = client .post(resource.token_url) .form(¶ms) .basic_auth(resource.client_id, Some(resource.client_secret)) @@ -438,8 +476,11 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn proxy_request_to_request_builder(proxy_request: ProxyRequest) -> RequestBuilder { - let mut request = HTTP_CLIENT.request(proxy_request.method.clone(), &proxy_request.url); +fn proxy_request_to_request_builder( + client: &Client, + proxy_request: ProxyRequest, +) -> RequestBuilder { + let mut request = client.request(proxy_request.method.clone(), &proxy_request.url); for (header_name, header_value) in &proxy_request.headers { request = request.header(header_name.as_str(), header_value.as_str()); } @@ -564,6 +605,8 @@ async fn global_proxy( custom_headers: HashMap::new(), }; + let client = pinned_ai_client_for(&credentials.base_url).await?; + if matches!(proxy_mode, ProxyExecutionMode::NativeGoogleAi) { let proxy_args = ProxyBuildArgs { method: &method, @@ -576,8 +619,8 @@ async fn global_proxy( audit_global_ai_request(&db, &authed).await?; let response = match ai_path.as_str() { - "chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await, - "models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await, + "chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await, + "models" => handle_google_ai_models_proxy(&client, &proxy_args).await, _ => Err(Error::BadRequest(format!( "Unsupported Google AI path: {}", ai_path @@ -600,7 +643,7 @@ async fn global_proxy( body: &body, credentials: &credentials, })?; - proxy_request_to_request_builder(proxy_request) + proxy_request_to_request_builder(&client, proxy_request) } ProxyExecutionMode::NativeGoogleAi | ProxyExecutionMode::NativeAwsBedrock => { return Err(Error::BadRequest(format!( @@ -882,9 +925,11 @@ async fn proxy( credentials: &credentials, }; + let client = pinned_ai_client_for(&credentials.base_url).await?; + let response = match ai_path.as_str() { - "chat/completions" => handle_google_ai_chat_proxy(&HTTP_CLIENT, &proxy_args).await, - "models" => handle_google_ai_models_proxy(&HTTP_CLIENT, &proxy_args).await, + "chat/completions" => handle_google_ai_chat_proxy(&client, &proxy_args).await, + "models" => handle_google_ai_models_proxy(&client, &proxy_args).await, _ => Err(Error::BadRequest(format!( "Unsupported Google AI path: {}", ai_path @@ -943,7 +988,8 @@ async fn proxy( body: &body, credentials: &credentials, })?; - proxy_request_to_request_builder(proxy_request) + let client = pinned_ai_client_for(&credentials.base_url).await?; + proxy_request_to_request_builder(&client, proxy_request) } ProxyExecutionMode::NativeGoogleAi => { return Err(Error::internal_err( diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 8169a42677..4a6d5134d2 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,7 +69,7 @@ use windmill_common::{ users::username_to_permissioned_as, utils::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - Pagination, RunnableKind, StripPath, + strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -383,11 +383,7 @@ async fn list_search_apps( // apps' definitions. `check_scopes` uses ScopeDefinition::includes, where run // does NOT include read, so it correctly denies such tokens. check_scopes(&authed, || "apps:read".to_string())?; - #[cfg(feature = "enterprise")] let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; let mut tx = user_db.begin(&authed).await?; let allowed = build_scope_path_predicate(&authed, "apps", "read"); @@ -1832,50 +1828,6 @@ fn custom_path_conflict_error( } } -/// App values live in a `json` column, which — unlike `jsonb` — accepts the -/// `\u0000` escape. Any later `json`→`jsonb` conversion (a workspace fork's -/// `clone_apps`, search indexing, …) then aborts with "unsupported Unicode -/// escape sequence". Strip genuine NULs so the value is jsonb-safe before it -/// lands in the DB; the usual source is a binary file such as `.DS_Store` -/// accidentally bundled into a raw app's file map. A real NUL is unstorable -/// either way, and frontend code that needs the character writes it as the -/// source escape `\u0000`, which JSON-encodes to `\\u0000` (an escaped -/// backslash — the even-parity case below) and is left untouched. -/// -/// Returns `Cow::Borrowed` (no allocation) when the value is already clean. -fn strip_null_chars(raw: &str) -> Cow<'_, str> { - let bytes = raw.as_bytes(); - let mut out: Option = None; - let mut copied_to = 0; - let mut search_from = 0; - // A genuine NUL is `\u0000`: a `u0000` introduced by an *odd* run of - // backslashes. An even run (`\\u0000`) is an escaped backslash then the - // literal text "u0000" (common in minified JS regexes) and is preserved. - while let Some(rel) = raw[search_from..].find("u0000") { - let at = search_from + rel; - let mut backslashes = 0; - let mut j = at; - while j > 0 && bytes[j - 1] == b'\\' { - backslashes += 1; - j -= 1; - } - if backslashes % 2 == 1 { - // Drop the escaping backslash + `u0000` — the 6 chars in [at-1, at+5). - let out = out.get_or_insert_with(String::new); - out.push_str(&raw[copied_to..at - 1]); - copied_to = at + 5; - } - search_from = at + 5; - } - match out { - Some(mut out) => { - out.push_str(&raw[copied_to..]); - Cow::Owned(out) - } - None => Cow::Borrowed(raw), - } -} - async fn create_app_internal<'a>( authed: ApiAuthed, db: sqlx::Pool, @@ -2021,7 +1973,7 @@ async fn create_app_internal<'a>( .await?; // `.get()` keeps the raw text (and thus key order); strip any NUL so the // `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it. - let value = strip_null_chars(app.value.0.get()); + let value = strip_json_nul(app.value.0.get()); if matches!(value, Cow::Owned(_)) { tracing::warn!(path = %app.path, "stripped NUL character(s) from app value on create"); } @@ -2606,7 +2558,7 @@ async fn update_app_internal<'a>( // `.get()` keeps the raw text (and thus key order); strip any NUL so the // `json`→`jsonb` conversion downstream (fork, indexing) can't choke on it. - let value = strip_null_chars(nvalue.0.get()); + let value = strip_json_nul(nvalue.0.get()); if matches!(value, Cow::Owned(_)) { tracing::warn!(path = %npath, "stripped NUL character(s) from app value on update"); } @@ -3883,6 +3835,18 @@ async fn get_on_behalf_authed_from_app( Ok((on_behalf_authed, policy)) } +/// Which identity a deployed `apps_u/*` S3 read runs as. +#[cfg(feature = "parquet")] +enum AppS3ReadIdentity { + /// The gate passed: read with the policy's on-behalf identity (the app author in + /// author-mode, the viewer in viewer-mode). + OnBehalf, + /// The gate did not pass but a logged-in, non-embed viewer is present: read with + /// the viewer's OWN identity so the downstream S3 permission check self-enforces + /// their entitlement (never the author's). + AsViewer(ApiAuthed), +} + #[cfg(feature = "parquet")] async fn check_if_allowed_to_access_s3_file_from_app( db: &DB, @@ -3891,11 +3855,15 @@ async fn check_if_allowed_to_access_s3_file_from_app( w_id: &str, path: &str, policy: &Policy, -) -> Result<()> { +) -> Result { let is_app_embed = opt_authed.as_ref().is_some_and(|authed| { windmill_api_auth::scopes::has_app_embed_sentinel(authed.scopes.as_deref()) }); + // A valid presigned bearer is a self-authorizing capability, so it short-circuits + // the provenance gate. OSS builds cannot validate signatures (no workspace-key + // HMAC), so there the bearer is ignored and the request falls through to the + // checks below — the same path these routes took before presigning. if file_query.sig.is_some() { #[cfg(feature = "private")] { @@ -3908,35 +3876,34 @@ async fn check_if_allowed_to_access_s3_file_from_app( &db, ) .await?; - Ok(()) + return Ok(AppS3ReadIdentity::OnBehalf); } - #[cfg(not(feature = "private"))] - return Err(Error::InternalErr( - "Internal error: signature validation is not supported in open source mode".to_string(), - )); - } else if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed { + } + + if matches!(policy.execution_mode, ExecutionMode::Viewer) && !is_app_embed { // Viewer mode: the on-behalf identity IS the viewer, so the downstream // get_workspace_s3_resource_and_check_paths already bounds the read by // their own perms — no provenance gate (it would over-restrict). Embed // tokens are excluded (untrusted app JS stays confined below). - Ok(()) - } else { - // Author-mode/embed: confine reads to the app's declared keys or files THIS - // app produced, else a viewer could launder the author's S3 perms via an - // arbitrary file_key (confused deputy). Provenance is the un-forgeable - // app-origination marker (`trigger_kind='app'` + `trigger=`); - // `created_by=` is ANDed only as a per-viewer isolation filter (it - // can narrow — one viewer can't read another's result — never forge). - let creator = opt_authed - .as_ref() - .map(|authed| authed.username.clone()) - .unwrap_or_else(|| "anonymous".to_string()); - let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| { - keys.iter() - .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) - }) || { - sqlx::query_scalar!( - r#"SELECT EXISTS ( + return Ok(AppS3ReadIdentity::OnBehalf); + } + + // Author-mode/embed: confine reads to the app's declared keys or files THIS + // app produced, else a viewer could launder the author's S3 perms via an + // arbitrary file_key (confused deputy). Provenance is the un-forgeable + // app-origination marker (`trigger_kind='app'` + `trigger=`); + // `created_by=` is ANDed only as a per-viewer isolation filter (it + // can narrow — one viewer can't read another's result — never forge). + let creator = opt_authed + .as_ref() + .map(|authed| authed.username.clone()) + .unwrap_or_else(|| "anonymous".to_string()); + let allowed = policy.allowed_s3_keys.as_ref().is_some_and(|keys| { + keys.iter() + .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) + }) || { + sqlx::query_scalar!( + r#"SELECT EXISTS ( SELECT 1 FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $2 AND c.started_at > now() - interval '3 hours' @@ -3945,21 +3912,45 @@ async fn check_if_allowed_to_access_s3_file_from_app( AND j.trigger = $3 AND j.created_by = $4 )"#, - file_query.s3, - w_id, - path, - creator, - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; + file_query.s3, + w_id, + path, + creator, + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; - if !allowed { - Err(Error::BadRequest("File restricted".to_string())) - } else { - Ok(()) + if allowed { + return Ok(AppS3ReadIdentity::OnBehalf); + } + + // Gate denied. A viewer whose token is effectively unscoped falls back to reading + // as THEMSELVES: the file is still bounded by their own S3 perms downstream, and + // such a token can already fetch it via `job_helpers/download_s3_file`, so the + // fallback adds zero capability. `is_effectively_unscoped` (the same predicate the + // route-scope middleware uses) is what makes that true: a genuinely scope-restricted + // token (e.g. `apps:read:`) is allowed on `apps_u/*` but REJECTED on + // `job_helpers/*`, so serving it the file here WOULD be a new capability — it stays + // gated. `!is_app_embed` keeps that confinement explicit (embed tokens carry the + // `app_embed` scope, so they are already scope-restricted). Anonymous callers (no + // identity) also have no viewer to fall back to. Only the confused-deputy denial + // reaches the message below. + match opt_authed.as_ref() { + Some(viewer) + if !is_app_embed + && windmill_api_auth::is_effectively_unscoped(viewer.scopes.as_deref()) => + { + Ok(AppS3ReadIdentity::AsViewer(viewer.clone())) } + _ => Err(Error::BadRequest(format!( + "S3 file \"{}\" is not accessible from this app. A deployed app running on \ + behalf of its author only serves files it generated, files in its declared \ + allowlist, or presigned files. To expose a pre-existing file, sign it \ + (signS3Object / sign_s3_object) or set the app's execution mode to \"viewer\".", + file_query.s3 + ))), } } @@ -4027,7 +4018,7 @@ async fn download_s3_file_from_app( get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, force_viewer_allowed_s3_keys) .await?; - check_if_allowed_to_access_s3_file_from_app( + let read_authed = match check_if_allowed_to_access_s3_file_from_app( &db, &opt_authed, &query.file_query, @@ -4035,10 +4026,14 @@ async fn download_s3_file_from_app( &path, &policy, ) - .await?; + .await? + { + AppS3ReadIdentity::OnBehalf => on_behalf_authed, + AppS3ReadIdentity::AsViewer(viewer) => viewer, + }; download_s3_file_internal( - OptJobAuthed { authed: on_behalf_authed, job_id: None }, + OptJobAuthed { authed: read_authed, job_id: None }, &db, None, &w_id, @@ -4051,14 +4046,25 @@ async fn download_s3_file_from_app( .await } +// Presigned bearer params (`exp=..&sig=..`) extracted as a second `Query` so the +// app-scoped preview/count/metadata routes honor a presigned key the same way the +// raw `download_s3_file` route does. #[cfg(feature = "parquet")] -fn app_s3_file_query(s3: String, storage: Option) -> AppS3FileQuery { +#[derive(Deserialize)] +struct AppS3Sig { + sig: Option, + #[cfg(feature = "private")] + exp: Option, +} + +#[cfg(feature = "parquet")] +fn app_s3_file_query(s3: String, storage: Option, sig: AppS3Sig) -> AppS3FileQuery { AppS3FileQuery { s3, storage, - sig: None, + sig: sig.sig, #[cfg(feature = "private")] - exp: None, + exp: sig.exp, } } @@ -4079,9 +4085,15 @@ async fn app_s3_on_behalf_and_provenance( } let (on_behalf_authed, policy) = get_on_behalf_authed_from_app(db, path, w_id, opt_authed, None).await?; - check_if_allowed_to_access_s3_file_from_app(db, opt_authed, file_query, w_id, path, &policy) - .await?; - Ok(crate::db::OptJobAuthed { authed: on_behalf_authed, job_id: None }) + let read_authed = match check_if_allowed_to_access_s3_file_from_app( + db, opt_authed, file_query, w_id, path, &policy, + ) + .await? + { + AppS3ReadIdentity::OnBehalf => on_behalf_authed, + AppS3ReadIdentity::AsViewer(viewer) => viewer, + }; + Ok(crate::db::OptJobAuthed { authed: read_authed, job_id: None }) } // The app-scoped display ops carry the app path in the URL and everything else @@ -4154,9 +4166,10 @@ async fn app_download_s3_parquet_file_as_csv( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; crate::job_helpers_oss::download_s3_parquet_file_as_csv_internal( @@ -4178,14 +4191,19 @@ async fn app_load_file_metadata( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, + Query(mut query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; + // On-behalf app reads are confined to the workspace storage; a + // viewer-supplied custom resource must not be honored. + query.s3_resource_path = None; let resp = - crate::job_helpers_oss::load_file_metadata_internal(job_authed, &db, &w_id, query).await?; + crate::job_helpers_oss::load_file_metadata_internal(job_authed, &db, None, &w_id, query) + .await?; Ok(Json(resp).into_response()) } @@ -4194,14 +4212,19 @@ async fn app_load_file_preview( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, + Query(mut query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); - let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone()); + let file_query = app_s3_file_query(query.file_key.clone(), query.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; + // On-behalf app reads are confined to the workspace storage; a + // viewer-supplied custom resource must not be honored. + query.s3_resource_path = None; let resp = - crate::job_helpers_oss::load_file_preview_internal(job_authed, &db, &w_id, query).await?; + crate::job_helpers_oss::load_file_preview_internal(job_authed, &db, None, &w_id, query) + .await?; Ok(Json(resp).into_response()) } @@ -4211,10 +4234,11 @@ async fn app_load_table_count( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = @@ -4229,10 +4253,11 @@ async fn app_load_parquet_preview( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( @@ -4248,10 +4273,11 @@ async fn app_load_csv_preview( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, + Query(sig): Query, ) -> Result { let path = path.to_path(); let (file_key, inner) = query.into_inner(); - let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone()); + let file_query = app_s3_file_query(file_key.clone(), inner.storage.clone(), sig); let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( @@ -4779,62 +4805,3 @@ mod embed_token_tests { assert!(parse_embed_policy("not json").is_err()); } } - -#[cfg(test)] -mod strip_null_chars_tests { - use super::strip_null_chars; - use std::borrow::Cow; - - // Build `{"k":"u0000"}` without writing the escape literally - // (a real NUL can't live in Rust source). Odd n => the trailing `u0000` is a - // genuine NUL escape; even n => an escaped backslash then the text "u0000". - fn doc(backslashes: usize) -> String { - format!(r#"{{"k":"{}u0000"}}"#, "\\".repeat(backslashes)) - } - - #[test] - fn strips_genuine_null_escape() { - // 1 backslash: the NUL escape is dropped, the string value becomes "". - assert_eq!(strip_null_chars(&doc(1)).as_ref(), r#"{"k":""}"#); - // 3 backslashes: escaped backslash + NUL -> keep the escaped backslash. - let three = doc(3); - let out = strip_null_chars(&three); - assert_eq!(out.as_ref(), r#"{"k":"\\"}"#); - // Result is now valid, NUL-free JSON (i.e. jsonb-safe). - let v: serde_json::Value = serde_json::from_str(out.as_ref()).unwrap(); - assert!(!v["k"].as_str().unwrap().as_bytes().contains(&0u8)); - } - - #[test] - fn preserves_escaped_backslash_then_literal_u0000() { - // Even runs are the literal text "u0000" (e.g. a minified JS regex char - // class) and must be returned untouched, with no allocation. - for n in [2usize, 4] { - let s = doc(n); - let out = strip_null_chars(&s); - assert_eq!(out.as_ref(), s.as_str()); - assert!(matches!(out, Cow::Borrowed(_)), "n={n} should be borrowed"); - } - } - - #[test] - fn preserves_clean_values() { - // Plain value, and the bare token "u0000" with no preceding backslash. - for s in [r#"{"files":{"/index.tsx":"hello"}}"#, r#"{"k":"u0000"}"#] { - let out = strip_null_chars(s); - assert_eq!(out.as_ref(), s); - assert!(matches!(out, Cow::Borrowed(_))); - } - // The escape for a literal backslash char (`u005c`) then text "u0000": - // the only "u0000" match is preceded by `c` (0 backslashes) -> no NUL. - let s = format!(r#"{{"k":"{}u005cu0000"}}"#, "\\"); - assert!(matches!(strip_null_chars(&s), Cow::Borrowed(_))); - } - - #[test] - fn strips_multiple_and_preserves_surrounding() { - // Mirrors the .DS_Store case: several NULs interleaved with real text. - let s = format!(r#"{{"a":"x{b}u0000{b}u0000y","b":"ok"}}"#, b = "\\"); - assert_eq!(strip_null_chars(&s).as_ref(), r#"{"a":"xy","b":"ok"}"#); - } -} diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index bf98ae460d..1eed038c25 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -48,6 +48,8 @@ use windmill_common::error::Error; #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] use crate::triggers::kafka::KafkaTriggerConfigConnection; +#[cfg(feature = "amqp_trigger")] +use crate::triggers::amqp::{AmqpOptions, ExchangeConfig}; #[cfg(feature = "mqtt_trigger")] use crate::triggers::mqtt::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; @@ -234,6 +236,14 @@ pub struct MqttTriggerConfig { pub client_version: Option, pub client_id: Option, } +#[cfg(feature = "amqp_trigger")] +#[derive(Debug, Serialize, Deserialize)] +pub struct AmqpTriggerConfig { + pub amqp_resource_path: String, + pub queue_name: String, + pub exchange: Option, + pub options: Option, +} #[cfg(feature = "postgres_trigger")] #[derive(Serialize, Deserialize, Debug)] pub struct PostgresTriggerConfig { @@ -271,6 +281,8 @@ enum TriggerConfig { Nats(NatsTriggerConfig), #[cfg(feature = "mqtt_trigger")] Mqtt(MqttTriggerConfig), + #[cfg(feature = "amqp_trigger")] + Amqp(AmqpTriggerConfig), #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] Gcp(GcpTriggerConfig), #[cfg(all(feature = "enterprise", feature = "azure_trigger", feature = "private"))] diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index c564548b42..aa94fd80c4 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -93,6 +93,9 @@ lazy_static::lazy_static! { (20260710073406, include_str!( "../../migrations/20260710073406_index_v2_job_parent_job.up.sql" ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), + (20260724094737, include_str!( + "../../migrations/20260724094737_runnables_sort_indexes.up.sql" + ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")), ].into_iter().collect(); } diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index ffc94d5b46..b44b0e64be 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -19,6 +19,7 @@ use windmill_common::{ error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, + utils::strip_json_nul, variables::{build_crypt, encrypt}, }; @@ -64,6 +65,11 @@ pub struct DraftListItem { /// so it defaults to `false` when read from the row. #[sqlx(default)] pub can_write: bool, + /// `true` when this draft is identical (jsonb-equal) to the parent's draft at + /// the same (path, kind, owner). `None` unless the request passed a valid + /// `compare_to_workspace`. + #[serde(skip_serializing_if = "Option::is_none")] + pub unchanged_from_parent: Option, /// The listed row belongs to the authed user (own draft or the legacy /// no-owner row) and is therefore actionable by them. Always `true` in the /// default (own-drafts) listing; only meaningful with `all_users=true`, @@ -77,6 +83,10 @@ pub struct ListDraftsQuery { /// List every draft in the workspace (all users), not just the authed /// user's own + legacy rows. Other users' rows come back with `mine=false`. pub all_users: Option, + /// A fork passes its parent workspace id here to have each row flagged with + /// `unchanged_from_parent`. Honored only when it is this workspace's actual + /// `parent_workspace_id` (enforced in `list_drafts`). + pub compare_to_workspace: Option, } /// Every draft the authed user has in this workspace, across all kinds — the @@ -97,9 +107,31 @@ async fn list_drafts( return Ok(Json(vec![])); } let all_users = query.all_users.unwrap_or(false); + // Only honor `compare_to_workspace` when it is genuinely this workspace's + // parent (a fork comparing against its source). Any other value is dropped + // so the value-equality subquery can't be used to probe an unrelated + // workspace's draft contents. + let compare_to_workspace = match &query.compare_to_workspace { + Some(candidate) => { + let parent: Option = sqlx::query_scalar::<_, Option>( + "SELECT parent_workspace_id FROM workspace WHERE id = $1", + ) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + if parent.as_deref() == Some(candidate.as_str()) { + Some(candidate.clone()) + } else { + None + } + } + None => None, + }; let rows = sqlx::query_as::<_, DraftListItem>(&list_drafts_query(all_users)) .bind(&w_id) .bind(&authed.email) + .bind(&compare_to_workspace) .fetch_all(&db) .await?; // Per-row permission gating: @@ -149,8 +181,10 @@ async fn list_drafts( /// `deployed_table()` (shared single source — can't drift from the access /// check). Table names come from the closed enum, never user input. Kinds /// with no path-keyed table get no arm and fall to `ELSE true`. -/// `$1` = workspace_id, `$2` = email. With `all_users` the owner filter is -/// dropped so every workspace draft is listed (others' rows get `mine=false`). +/// `$1` = workspace_id, `$2` = email, `$3` = the parent workspace to compare +/// against (nullable; drives `unchanged_from_parent`). With `all_users` the +/// owner filter is dropped so every workspace draft is listed (others' rows get +/// `mine=false`). fn list_drafts_query(all_users: bool) -> String { let mut case = String::from("CASE d.typ::text\n"); for kind in UserDraftItemKind::ALL { @@ -221,7 +255,18 @@ fn list_drafts_query(all_users: bool) -> String { ) AS draft_path, (d.email IS NULL) AS legacy_draft, (d.email = $2 OR d.email IS NULL) AS mine, - {case} AS draft_only + {case} AS draft_only, + -- value is a `json` column (no `=` operator), so compare as jsonb. + CASE WHEN $3::text IS NULL THEN NULL::bool + ELSE EXISTS( + SELECT 1 FROM draft pd + WHERE pd.workspace_id = $3 + AND pd.path = d.path + AND pd.typ = d.typ + AND pd.email IS NOT DISTINCT FROM d.email + AND pd.value::jsonb = d.value::jsonb + ) + END AS unchanged_from_parent FROM draft d WHERE d.workspace_id = $1{owner_filter} ORDER BY d.path, d.typ, @@ -309,7 +354,7 @@ async fn update_draft( // `draft.value` is a `json` column, so a U+0000 (NUL) would persist as an // escape and later make any `->>`/`to_jsonb` extraction raise `22P05`. // Strip it here so a NUL never reaches the column. - let serialized = strip_json_nul(serialized); + let serialized = strip_json_nul(&serialized); // Upsert. The conflict check rides on the DO UPDATE WHERE clause — // when the row is newer than `last_sync`, RETURNING yields nothing. // `created_at` defaults to `now()` but the migration overrides it ($8) @@ -327,7 +372,7 @@ async fn update_draft( email, path, kind as UserDraftItemKind, - serialized, + serialized.as_ref(), req.last_sync, req.force, req.created_at, @@ -477,53 +522,6 @@ async fn migrate_legacy_draft( } } -/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to -/// store in the `json`-typed `draft.value` (a NUL there would later make any -/// `->>`/`to_jsonb` extraction raise `22P05`). -/// -/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a -/// backslash only ever occurs inside a string, so one backslash-parity-aware -/// pass removes every real NUL escape — covering values and keys alike — while -/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal -/// text `u0000`) intact. O(n) over the bytes with no `serde_json::Value` tree to -/// allocate, and the fast path (no such substring at all) returns the input -/// untouched. The slow path is reached not only by genuinely poisoned values but -/// by any value that legitimately contains `u0000` after a backslash (e.g. script -/// source), so it must stay allocation-light for potentially large drafts. -fn strip_json_nul(serialized: String) -> String { - if !serialized.contains("\\u0000") { - return serialized; - } - let bytes = serialized.as_bytes(); - let mut out: Vec = Vec::with_capacity(bytes.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] != b'\\' { - out.push(bytes[i]); - i += 1; - continue; - } - // Consume the whole run of backslashes. An even run is N/2 escaped - // backslashes and leaves the next char unescaped; an odd run ends in an - // escaping backslash, so a following `u0000` is a real NUL escape. - let run_start = i; - while i < bytes.len() && bytes[i] == b'\\' { - i += 1; - } - let run = i - run_start; - if run % 2 == 1 && bytes[i..].starts_with(b"u0000") { - // Drop the escaping backslash + `u0000`; keep the leading literal pairs. - out.extend(std::iter::repeat(b'\\').take(run - 1)); - i += 5; - } else { - out.extend(std::iter::repeat(b'\\').take(run)); - } - } - // Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain - // valid UTF-8 (and valid JSON). - String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8") -} - /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -815,65 +813,3 @@ async fn require_can_read_path( } Err(Error::NotFound(format!("no draft visible at {path}"))) } - -#[cfg(test)] -mod tests { - use super::strip_json_nul; - - // Parse the (NUL-free) result so assertions read clearly. - fn parsed(s: String) -> serde_json::Value { - serde_json::from_str(&s).expect("strip_json_nul must return valid JSON") - } - - #[test] - fn clean_value_is_returned_byte_for_byte() { - let s = r#"{"summary":"all good","n":1}"#.to_string(); - assert_eq!(strip_json_nul(s.clone()), s); - } - - #[test] - fn real_nul_in_value_is_stripped() { - let out = strip_json_nul(r#"{"summary":"hi\u0000there"}"#.to_string()); - assert!(!out.contains(r"\u0000")); - assert_eq!(parsed(out)["summary"], "hithere"); - } - - #[test] - fn legit_escaped_backslash_is_a_noop() { - // JSON "a\\u0000b" decodes to the 8-char string a,backslash,u,0,0,0,0,b - // — not a NUL — so the value is already clean and round-trips byte-for-byte. - let s = r#"{"summary":"a\\u0000b"}"#.to_string(); - assert_eq!(strip_json_nul(s.clone()), s); - } - - #[test] - fn collision_real_and_literal_both_handled() { - // "a" carries a real NUL; "b" carries the literal text backslash-u0000. - // The value walk strips the former and leaves the latter intact — the - // pathological case that needed a fallback in SQL is trivial in Rust. - let v = parsed(strip_json_nul( - r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string(), - )); - assert_eq!(v["a"], "xy"); - assert_eq!(v["b"], "p\\u0000q"); - } - - #[test] - fn nested_values_and_keys_are_cleaned() { - let out = strip_json_nul( - r#"{"o":{"k\u0000":["a\u0000b",{"deep\u0000":"v\u0000"}]}}"#.to_string(), - ); - assert!(!out.contains(r"\u0000")); - let v = parsed(out); - assert_eq!(v["o"]["k"][0], "ab"); - assert_eq!(v["o"]["k"][1]["deep"], "v"); - } - - #[test] - fn odd_backslash_run_keeps_literal_drops_nul() { - // JSON "a\\\u0000b" is an escaped backslash (kept) immediately followed by - // a real NUL escape (dropped) -> decodes to a,backslash,b. - let v = parsed(strip_json_nul(r#"{"x":"a\\\u0000b"}"#.to_string())); - assert_eq!(v["x"], "a\\b"); - } -} diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs new file mode 100644 index 0000000000..9c1412cb0b --- /dev/null +++ b/backend/windmill-api/src/hub_publish.rs @@ -0,0 +1,622 @@ +use crate::auth::Tokened; +use crate::db::ApiAuthed; +use crate::HTTP_CLIENT; +use axum::{ + extract::{DefaultBodyLimit, FromRequestParts, Json, Path, Query, RawPathParams}, + http::{request::Parts, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use serde::{Deserialize, Deserializer, Serialize}; +use windmill_common::{ + error::{to_anyhow, Error}, + utils::require_admin, + HUB_BASE_URL, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/publish_draft", post(publish_draft)) + .route("/scripts", post(publish_script)) + .route("/flows", post(publish_flow)) + .route("/apps", post(publish_app)) + .route("/raw_apps", post(publish_raw_app)) + .route("/raw_apps/{id}/embed", post(publish_raw_app_embed)) + .route( + "/scripts/{ask_id}/recording", + post(publish_script_recording), + ) + .route("/flows/{flow_id}/recording", post(publish_flow_recording)) + .route( + "/projects/{slug}/pipeline_recording", + post(publish_pipeline_recording), + ) + .route( + "/projects/{slug}/logo", + post(publish_project_logo).layer(DefaultBodyLimit::max(LOGO_BODY_LIMIT)), + ) + .route("/resource_types", post(publish_resource_type)) + .route("/resources", post(publish_resources)) + .route("/triggers", post(publish_triggers)) + .route("/migrations", post(publish_migrations)) + .route("/projects/{slug}/export", get(get_project_export)) + .route("/projects/{slug}/submit", post(submit_project)) + .route("/project", get(get_project_by_source)) +} + +#[derive(Deserialize)] +struct HubScope { + folder: Option, +} + +fn validate_folder(folder: &str) -> Result<(), Error> { + let ok = !folder.is_empty() + && folder.len() <= 255 + && folder + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid folder: {folder}"))) + } +} + +fn source_key(workspace: &str, folder: &str) -> Result { + validate_folder(folder)?; + Ok(format!("{workspace}:{folder}")) +} + +fn validate_project_slug(slug: &str) -> Result<(), Error> { + let ok = slug.len() >= 3 + && slug.len() <= 50 + && !slug.starts_with('-') + && !slug.ends_with('-') + && slug + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid project slug: {slug}"))) + } +} + +/// A Hub project slug that is valid by construction: deserialization (from a +/// request body or a path segment) is the only way to obtain one and it runs +/// `validate_project_slug`, so no handler can forward or interpolate an +/// unvalidated slug into a Hub URL. +#[derive(Serialize)] +#[serde(transparent)] +struct ProjectSlug(String); + +impl<'de> Deserialize<'de> for ProjectSlug { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + validate_project_slug(&s).map_err(serde::de::Error::custom)?; + Ok(ProjectSlug(s)) + } +} + +impl std::fmt::Display for ProjectSlug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// The single gate every Hub endpoint goes through: an admin caller, their +/// token (the Hub authenticates it back against this instance's whoami), and +/// the validated `workspace_id:folder` source key scoping ownership Hub-side. +/// Handlers can only reach the Hub via this extractor's methods, so a new +/// endpoint cannot forget the admin check or folder validation. +/// +/// A workspace can publish one Hub project per folder. The stable, never-mutated +/// link key is `workspace_id:folder_name` (folder name is the path segment and is +/// never renamed — only display_name changes). `:` is safe: neither workspace ids +/// nor folder names (alphanumeric, underscore, hyphen) contain it. +struct HubPublishCtx { + source_id: Option, + token: String, +} + +impl FromRequestParts for HubPublishCtx +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + let authed = ApiAuthed::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let tokened = Tokened::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let params = RawPathParams::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let workspace = params + .iter() + .find(|(k, _)| *k == "workspace_id") + .map(|(_, v)| v.to_owned()); + let Query(scope) = Query::::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let build = || -> Result { + require_admin(authed.is_admin, &authed.username)?; + let workspace = workspace.ok_or_else(|| { + Error::internal_err( + "hub publish route must be nested under /w/{workspace_id}".to_string(), + ) + })?; + let source_id = scope + .folder + .as_deref() + .map(|f| source_key(&workspace, f)) + .transpose()?; + Ok(HubPublishCtx { source_id, token: tokened.token }) + }; + build().map_err(IntoResponse::into_response) + } +} + +impl HubPublishCtx { + fn require_source(&self) -> Result<&str, Error> { + self.source_id + .as_deref() + .ok_or_else(|| Error::BadRequest("missing folder query param".to_string())) + } + + async fn post( + &self, + path: &str, + body: &T, + ) -> Result<(StatusCode, String), Error> { + forward_to_hub(path, self.require_source()?, &self.token, body).await + } + + async fn get(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.require_source()?, &self.token).await + } + + /// GET without requiring a folder scope. Only for reads the Hub allows + /// publicly (e.g. exporting an approved project); the empty source id makes + /// the Hub skip the ownership match. + async fn get_maybe_unscoped(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.source_id.as_deref().unwrap_or(""), &self.token).await + } +} + +#[derive(Deserialize, Serialize)] +struct PublishDraftBody { + slug: ProjectSlug, + name: String, + summary: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + readme: Option, +} + +async fn publish_draft( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/projects", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishScriptBody { + summary: String, + app: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + kind: Option, + content: String, + language: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lockfile: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_script( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/scripts/add", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowInner { + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + value: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowBody { + flow: PublishFlowInner, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_flow( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/flows", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishAppBody { + app: serde_json::Value, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishRawAppBody { + raw: String, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/raw_apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct RawAppEmbedBody { + // No skip_serializing_if: `null` must reach the Hub to clear the embed (unpublish). + external_embed_url: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app_embed( + ctx: HubPublishCtx, + Path((_workspace, id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/raw_apps/{}/embed", id), &body).await +} + +#[derive(Deserialize, Serialize)] +struct RecordingBody { + #[serde(skip_serializing_if = "Option::is_none")] + recording: Option, + project_slug: ProjectSlug, +} + +async fn publish_script_recording( + ctx: HubPublishCtx, + Path((_workspace, ask_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/scripts/{}/recording", ask_id), &body) + .await +} + +async fn publish_flow_recording( + ctx: HubPublishCtx, + Path((_workspace, flow_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/flows/{}/recording", flow_id), &body) + .await +} + +// A data-pipeline recording is scoped to the whole project (a folder cascade), +// not a single Hub item, so the slug comes from the path (validated by +// construction) and only the opaque recording is forwarded. +#[derive(Deserialize, Serialize)] +struct PipelineRecordingBody { + #[serde(skip_serializing_if = "Option::is_none")] + recording: Option, +} + +async fn publish_pipeline_recording( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/pipeline_recording", slug), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct ProjectLogoInner { + b64: String, + mime: String, +} + +// Custom project logo (png/svg, base64). Double-Option so a missing `logo` +// key is distinguishable from an explicit `logo: null` (which clears the +// logo on the Hub) — otherwise POSTing `{}` would silently delete it. +#[derive(Deserialize, Serialize)] +struct ProjectLogoBody { + #[serde(default, deserialize_with = "deserialize_explicit")] + logo: Option>, +} + +fn deserialize_explicit<'de, D: Deserializer<'de>>( + d: D, +) -> Result>, D::Error> { + Option::::deserialize(d).map(Some) +} + +// Mirrors the Hub's own limits so an oversized/invalid payload is rejected +// here instead of being deserialized, copied and forwarded first. The route +// also carries a DefaultBodyLimit sized for a max logo in base64 + JSON +// envelope, overriding the much larger global request limit. +const MAX_LOGO_BYTES: usize = 512 * 1024; +const LOGO_BODY_LIMIT: usize = MAX_LOGO_BYTES / 3 * 4 + 16 * 1024; +const ALLOWED_LOGO_MIMES: [&str; 2] = ["image/png", "image/svg+xml"]; + +fn validate_logo(inner: &ProjectLogoInner) -> Result<(), Error> { + if !ALLOWED_LOGO_MIMES.contains(&inner.mime.as_str()) { + return Err(Error::BadRequest( + "logo mime must be image/png or image/svg+xml".to_string(), + )); + } + let b = inner.b64.as_bytes(); + let padding = b.iter().rev().take_while(|&&c| c == b'=').count(); + let valid = !b.is_empty() + && b.len() % 4 == 0 + && padding <= 2 + && b[..b.len() - padding] + .iter() + .all(|&c| c.is_ascii_alphanumeric() || c == b'+' || c == b'/'); + if !valid { + return Err(Error::BadRequest("invalid base64".to_string())); + } + if b.len() / 4 * 3 - padding > MAX_LOGO_BYTES { + return Err(Error::BadRequest(format!( + "logo too large (max {}KB)", + MAX_LOGO_BYTES / 1024 + ))); + } + Ok(()) +} + +async fn publish_project_logo( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, + Json(body): Json, +) -> Result { + let Some(logo) = body.logo else { + return Err(Error::BadRequest( + "logo field is required: an object to set it, or null to clear it".to_string(), + )); + }; + if let Some(inner) = &logo { + validate_logo(inner)?; + } + ctx.post( + &format!("/projects/{}/logo", slug), + &serde_json::json!({ "logo": logo }), + ) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceTypeBody { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + project_slug: ProjectSlug, +} + +async fn publish_resource_type( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/resource_types", body.project_slug), + &body, + ) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceBody { + path: String, + resource_type: String, +} + +#[derive(Deserialize, Serialize)] +struct PublishResourcesBody { + resources: Vec, + project_slug: ProjectSlug, +} + +async fn publish_resources( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/resources", body.project_slug), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggerBody { + path: String, + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + config: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + script_ask_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_id: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggersBody { + triggers: Vec, + project_slug: ProjectSlug, +} + +async fn publish_triggers( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/triggers", body.project_slug), &body) + .await +} + +// One best-effort data table migration attached to a project (per data table). +#[derive(Deserialize, Serialize)] +struct PublishMigrationBody { + datatable_name: String, + sql: String, + #[serde(default)] + sql_down: String, + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +struct PublishMigrationsBody { + migrations: Vec, + project_slug: ProjectSlug, +} + +async fn publish_migrations( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/migrations", body.project_slug), + &body, + ) + .await +} + +// Export is owner-scoped only when re-exporting your own draft; approved +// projects are public, so the folder scope is optional here. +async fn get_project_export( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.get_maybe_unscoped(&format!("/projects/{}/export", slug)) + .await +} + +async fn get_project_by_source(ctx: HubPublishCtx) -> Result { + ctx.get("/projects/by_source").await +} + +async fn submit_project( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/submit", slug), + &serde_json::json!({}), + ) + .await +} + +// The Hub has no auth of its own: it validates bearer tokens by calling this +// instance's /api/users/whoami. Forwarding the caller's own token logs them in +// on the Hub as themselves (account auto-created on first use). +async fn get_from_hub( + path: &str, + source_id: &str, + token: &str, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let res = HTTP_CLIENT + .get(&url) + .query(&[("source_id", source_id)]) + .bearer_auth(token) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + +async fn forward_to_hub( + path: &str, + source_id: &str, + token: &str, + body: &T, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let mut payload = serde_json::to_value(body).map_err(to_anyhow)?; + let obj = payload + .as_object_mut() + .ok_or_else(|| Error::internal_err("hub publish body must be a JSON object".to_string()))?; + obj.insert( + "source_id".to_string(), + serde_json::Value::String(source_id.to_string()), + ); + + let res = HTTP_CLIENT + .post(&url) + .bearer_auth(token) + .json(&payload) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 8a50f2e481..6cb7a4c769 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -200,7 +200,7 @@ pub async fn read_object_streamable( pub async fn delete_s3_file_internal( _authed: OptJobAuthed, _db: &DB, - _token: &str, + _user_db: Option, _w_id: &str, _query: DeleteS3FileQuery, ) -> error::Result<()> { @@ -215,6 +215,7 @@ pub async fn delete_s3_file_internal( pub struct DeleteS3FileQuery { pub file_key: String, pub storage: Option, + pub s3_resource_path: Option, } // Stubs for the app-scoped S3 display ops (mirrors the EE `*_internal` helpers + @@ -231,6 +232,7 @@ mod app_s3_display_stubs { pub struct LoadFileMetadataQuery { pub file_key: String, pub storage: Option, + pub s3_resource_path: Option, } #[derive(Serialize)] @@ -242,6 +244,7 @@ mod app_s3_display_stubs { #[allow(dead_code)] pub struct LoadFilePreviewQuery { pub storage: Option, + pub s3_resource_path: Option, pub file_key: String, pub file_size_in_bytes: Option, pub file_mime_type: Option, @@ -281,6 +284,7 @@ mod app_s3_display_stubs { pub async fn load_file_metadata_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _query: LoadFileMetadataQuery, ) -> error::Result { @@ -292,6 +296,7 @@ mod app_s3_display_stubs { pub async fn load_file_preview_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _query: LoadFilePreviewQuery, ) -> error::Result { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 14cbb76c91..2413ef47ff 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -25,7 +25,7 @@ use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; use url::Url; -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] use windmill_common::auth::is_super_admin_email; use windmill_common::auth::TOKEN_PREFIX_LEN; #[cfg(feature = "run_inline")] @@ -54,7 +54,7 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; use windmill_common::DYNAMIC_INPUT_CACHE; -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; use windmill_object_store::upload_artifact_to_store; #[cfg(feature = "run_inline")] @@ -342,6 +342,10 @@ pub fn workspaced_service() -> Router { "/resume_urls/{job_id}/{resume_id}", get(get_resume_urls).layer(cors.clone()), ) + .route( + "/wac_approval_urls/{job_id}/{step_key}", + get(get_wac_approval_urls).layer(cors.clone()), + ) .route( "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), @@ -1692,7 +1696,7 @@ impl<'a> GetQuery<'a> { } } -#[cfg(all(feature = "smtp", feature = "enterprise"))] +#[cfg(all(feature = "instance_smtp", feature = "enterprise"))] async fn send_workspace_trigger_failure_email_notification( db: &DB, w_id: &str, @@ -1855,7 +1859,7 @@ struct SendEmail { error: Value, } -#[cfg(all(feature = "enterprise", feature = "smtp"))] +#[cfg(all(feature = "enterprise", feature = "instance_smtp"))] async fn send_email_with_instance_smtp( authed: ApiAuthed, Extension(db): Extension, @@ -1913,7 +1917,7 @@ async fn send_email_with_instance_smtp( Ok(Json(resp)) } -#[cfg(not(all(feature = "enterprise", feature = "smtp")))] +#[cfg(not(all(feature = "enterprise", feature = "instance_smtp")))] async fn send_email_with_instance_smtp( _authed: ApiAuthed, Extension(_db): Extension, @@ -3192,17 +3196,9 @@ async fn resume_suspended( } // Check approval conditions - let approval_conditions = if is_wac { - flow.flow_status - .as_ref() - .and_then(|v| v.get("approval_conditions")) - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - } else { - flow.flow_status - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .and_then(|fs| fs.approval_conditions) - }; + let approval_conditions = extract_approval_conditions(flow.flow_status.as_ref(), is_wac); + + let trigger_email = flow.email.as_deref().unwrap_or(""); if let Some(ref ac) = approval_conditions { if ac.user_auth_required && opt_authed.is_none() { @@ -3214,6 +3210,13 @@ async fn resume_suspended( // If logged in, check authorization rules if let Some(ref authed) = opt_authed { + // self_approval_disabled applies to owners too (only admins are exempt), so it is + // enforced before the owner shortcut below. A token-only (anonymous) resume is treated as + // capability-based and intentionally not gated here; see resume_suspended_job. + if let Some(ref ac) = approval_conditions { + require_not_self_approval(authed, ac, trigger_email)?; + } + let is_admin = authed.is_admin; let is_owner = flow .script_path @@ -3222,7 +3225,6 @@ async fn resume_suspended( .unwrap_or(false); if !is_admin && !is_owner { - let trigger_email = flow.email.as_deref().unwrap_or(""); conditionally_require_authed_user( Some(authed.clone()), approval_conditions.clone(), @@ -3347,10 +3349,11 @@ struct ApprovalInfo { } /// Whether `opt_authed` is allowed to approve — and therefore view — this approval step. -/// Mirrors the authorization performed at the resume boundary: workspace admins and owners -/// of the runnable always qualify; otherwise the approval conditions (user_auth_required / -/// user_groups_required / self_approval_disabled) decide. When the step does not require auth, -/// an anonymous (token-only) caller qualifies. +/// Mirrors the authorization performed at the resume boundary: workspace admins always qualify; +/// self_approval_disabled then bars the triggerer even when they own the runnable; otherwise +/// owners qualify and the remaining approval conditions (user_auth_required / +/// user_groups_required) decide. When the step does not require auth, an anonymous (token-only) +/// caller qualifies. fn can_approve_step( opt_authed: &Option, approval_conditions: &Option, @@ -3362,6 +3365,12 @@ fn can_approve_step( if authed.is_admin { return true; } + // self_approval_disabled applies to owners too, so it gates the owner shortcut. + if let Some(ref ac) = approval_conditions { + if require_not_self_approval(authed, ac, trigger_email).is_err() { + return false; + } + } let is_owner = script_path .map(|p| require_owner_of_path(authed, p).is_ok()) .unwrap_or(false); @@ -3648,8 +3657,10 @@ async fn resume_suspended_job_internal( // Get flow info - works for step-level, flow-level, and WAC approval let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; - // HMAC secret = full capability. Skip approval_conditions checks. - // Authorization rules are enforced by the new resume_suspended endpoint instead. + // HMAC secret = full capability. Skip approval_conditions checks: possession of the full + // resume URL is the authorization (it is only disclosed to intended approvers, e.g. when a + // step returns it). Identity-based rules, including self_approval_disabled, are enforced by + // the resume_suspended endpoint instead. let exists = sqlx::query_scalar!( r#" @@ -3677,6 +3688,13 @@ async fn resume_suspended_job_internal( }; let mut tx: Transaction<'_, Postgres> = db.begin().await?; + // Inside the transaction that inserts the row and moves the suspend counter: + // validating earlier would let the workflow resolve this step and suspend on the + // next one in between, so a stale request would wake that later step instead. + if is_wac { + reject_mismatched_wac_approval(&mut tx, flow_info.id, resume_id).await?; + } + insert_resume_job( resume_id, job_id, @@ -3696,15 +3714,17 @@ async fn resume_suspended_job_internal( .execute(&mut *tx) .await?; } else if is_wac { - // WAC approval: decrement suspend counter directly on the WAC parent job - if flow_info.suspend > 0 { - sqlx::query!( - "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", - flow_info.id, - ) - .execute(&mut *tx) - .await?; - } + // WAC approval: decrement suspend counter directly on the WAC parent job. + // `flow_info.suspend` was read before this transaction took the queue-row + // lock, so gating on it would skip the decrement for a workflow that + // suspended in between and leave the approval parked until timeout. + sqlx::query!( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 AND suspend > 0", + flow_info.id, + ) + .execute(&mut *tx) + .await?; } else if is_flow_level { // For flow-level resumes, decrement the suspend counter if the flow is currently suspended // The approval will be matched when the worker checks for resumes (both step-level and flow-level) @@ -4095,6 +4115,45 @@ pub async fn get_suspended_job_flow( Ok(Json(SuspendedJobFlow { job: flow, approvers, view_token }).into_response()) } +/// Read the step's approval_conditions from the suspended flow status. For classic flows they +/// live inside the deserialized `FlowStatus`; for workflow-as-code they are a top-level +/// `approval_conditions` key in the status JSON. +fn extract_approval_conditions( + flow_status: Option<&serde_json::Value>, + is_wac: bool, +) -> Option { + if is_wac { + flow_status + .and_then(|v| v.get("approval_conditions")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + } else { + flow_status + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .and_then(|fs| fs.approval_conditions) + } +} + +/// The flow's triggerer may not approve their own suspended step when the step sets +/// `self_approval_disabled`. Only admins are exempt: owning the runnable does not grant +/// the right to approve your own run, so this must be enforced at every resume boundary +/// independently of the owner shortcut (which only waives user_auth_required / +/// user_groups_required). +fn require_not_self_approval( + authed: &ApiAuthed, + approval_conditions: &ApprovalConditions, + trigger_email: &str, +) -> error::Result<()> { + if approval_conditions.self_approval_disabled + && !authed.is_admin + && authed.email.eq(trigger_email) + { + return Err(Error::PermissionDenied( + "Self-approval is disabled for this flow step".to_string(), + )); + } + Ok(()) +} + fn conditionally_require_authed_user( _authed: Option, approval_conditions_opt: Option, @@ -4106,14 +4165,8 @@ fn conditionally_require_authed_user( let approval_conditions = approval_conditions_opt.unwrap(); // Check self-approval independently of user_auth_required - if approval_conditions.self_approval_disabled { - if let Some(ref authed) = _authed { - if !authed.is_admin && authed.email.eq(_trigger_email) { - return Err(Error::PermissionDenied( - "Self-approval is disabled for this flow step".to_string(), - )); - } - } + if let Some(ref authed) = _authed { + require_not_self_approval(authed, &approval_conditions, _trigger_email)?; } if approval_conditions.user_auth_required { @@ -4268,6 +4321,180 @@ pub async fn get_resume_urls( .await } +/// Resume URLs bound to one `wait_for_approval(key=...)` step of a running +/// Workflow-as-Code job, so the workflow can route the request through its own +/// channel instead of the built-in ones. Same authority as `get_resume_urls`: +/// only the `resume_id` derivation differs, and it is the one the worker will +/// use when that step suspends. +pub async fn get_wac_approval_urls( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, job_id, step_key)): Path<(String, Uuid, String)>, + Query(approver): Query, +) -> error::JsonResult { + if step_key.trim().is_empty() { + return Err(Error::BadRequest( + "step_key must be the key of a wait_for_approval step".to_string(), + )); + } + let flow_path = resume_target_flow_path(&db, &w_id, job_id).await?; + check_scopes(&authed, || format!("jobs:run:flows:{}", flow_path))?; + + // This handler writes to the job's status row, so the job must actually be in + // the caller's workspace — `v2_job_status` is keyed by job id alone and would + // otherwise take a write aimed at another workspace's job. + let in_workspace = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2)", + job_id, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !in_workspace { + return Err(Error::NotFound(format!("job {job_id} not found"))); + } + + // The approval belongs to the WAC parent, but WM_JOB_ID is the child job when + // this is called from inside a task() rather than a step(). Resolve up so the + // URL still targets the workflow that will suspend. + let job_id = get_flow_id_for_job(&db, job_id).await.unwrap_or(job_id); + + // The write below is what the run page keys its WAC timeline off, so it must not + // land on a job that has no WAC status. Rules out flows and step/child jobs; a + // WAC parent is itself a script job, so a plain script is indistinguishable here + // and still passes — it simply never mints, since only the SDK calls this. + let is_wac = sqlx::query_scalar!( + r#"SELECT (kind::text NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') + AND parent_job IS NULL) AS "is_wac!" + FROM v2_job WHERE id = $1"#, + job_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if !is_wac { + return Err(Error::BadRequest(format!( + "job {job_id} is not a workflow-as-code job" + ))); + } + + let resume_id = windmill_common::wac::approval_resume_id(&step_key); + + // Remember which steps have a minted URL in circulation. A workflow may mint + // several up front, and the resume path uses this to tell "URL for the step + // awaiting approval" apart from "URL for some other step of this workflow", + // which it otherwise cannot: the interactive channels sign random resume_ids + // and must keep resuming whatever step is pending. + // + // Record first, then look for a collision, both in one transaction: the upsert + // takes the row lock, so a concurrent mint of a colliding key is serialized + // behind it and sees this key rather than racing past an earlier read. Upsert + // because a workflow can mint before any step has checkpointed, and a bare + // UPDATE would silently match nothing and leave the link unbound. + let mut tx = db.begin().await?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_minted_approval_keys', + jsonb_build_object($2::text, true))) + ON CONFLICT (id) DO UPDATE SET workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + ARRAY['_minted_approval_keys'], + COALESCE(v2_job_status.workflow_as_code_status->'_minted_approval_keys', '{}'::jsonb) + || jsonb_build_object($2::text, true) + )", + ) + .bind(job_id) + .bind(&step_key) + .execute(&mut *tx) + .await?; + + // Two keys sharing a resume_id share one resume_job row and one capability, and + // the binding check could not tell which of them a link was minted for. + if let Some(other) = sqlx::query_scalar::<_, String>( + "SELECT jsonb_object_keys( + COALESCE(workflow_as_code_status->'_minted_approval_keys', '{}'::jsonb)) + FROM v2_job_status WHERE id = $1", + ) + .bind(job_id) + .fetch_all(&mut *tx) + .await? + .into_iter() + .find(|k| *k != step_key && windmill_common::wac::approval_resume_id(k) == resume_id) + { + tx.rollback().await?; + return Err(Error::BadRequest(format!( + "step key `{step_key}` collides with `{other}` on the same resume id; rename one" + ))); + } + tx.commit().await?; + + get_resume_urls_internal( + Extension(db), + Path((w_id, job_id, resume_id)), + Query(approver), + ) + .await +} + +/// A WAC resume URL minted for a named `wait_for_approval` step is accepted only +/// while that step is the one awaiting approval. Approval rows are consumed +/// oldest-first regardless of resume_id (WIN-2241 — required so Slack/Teams/the +/// approval page, which sign random ids, keep working), so a row banked at any +/// other moment is picked up by whichever approval is reached first, silently +/// answering it with this approver's response. Unbound resume_ids are untouched. +async fn reject_mismatched_wac_approval( + tx: &mut Transaction<'_, Postgres>, + job_id: Uuid, + resume_id: u32, +) -> Result<(), Error> { + // Lock the queue row the worker also writes when it suspends on the next step, + // so the pending step read below cannot change before this transaction commits. + sqlx::query("SELECT 1 FROM v2_job_queue WHERE id = $1 FOR UPDATE") + .bind(job_id) + .fetch_optional(&mut **tx) + .await?; + + let status: Option> = sqlx::query_scalar( + "SELECT jsonb_build_object( + 'minted', COALESCE(workflow_as_code_status->'_minted_approval_keys', '{}'::jsonb), + 'pending', workflow_as_code_status->'_checkpoint'->'pending_steps' + ) FROM v2_job_status WHERE id = $1", + ) + .bind(job_id) + .fetch_optional(&mut **tx) + .await?; + + let Some(sqlx::types::Json(binding)) = status else { + return Ok(()); + }; + let awaiting = binding.pending.as_ref().filter(|p| p.mode == "approval"); + let bound_to = binding + .minted + .keys() + .find(|k| windmill_common::wac::approval_resume_id(k) == resume_id); + + // A bound link is only ever valid while its own step is the one awaiting + // approval. Accepting it at any other time — including while the workflow is + // still running toward that step — leaves a row that the next approval to be + // reached consumes, whichever step that is. + match (bound_to, awaiting) { + (Some(step), pending) if !pending.is_some_and(|p| p.keys.iter().any(|k| k == step)) => { + Err(Error::BadRequest(format!( + "this approval link is bound to step `{step}`, which is not currently awaiting \ + approval" + ))) + } + _ => Ok(()), + } +} + +#[derive(Deserialize)] +struct WacApprovalBinding { + minted: std::collections::HashMap, + pending: Option, +} + pub async fn get_resume_urls_internal( Extension(db): Extension, Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 78dc97ca7d..dc5271f93d 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -97,12 +97,14 @@ mod granular_acls; mod group_history; mod groups; mod health; +mod hub_publish; #[cfg(feature = "private")] pub mod indexer_ee; mod indexer_oss; mod integration; mod internal_db; mod live_migrations; +mod runnables; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; @@ -573,6 +575,7 @@ pub async fn run_server( .nest("/embeddings", embeddings::workspaced_service()) .nest("/favorites", favorite::workspaced_service()) .nest("/flows", flows::workspaced_service()) + .nest("/runnables", runnables::workspaced_service()) .nest( "/workspace_dependencies", workspace_dependencies::workspaced_service(), @@ -659,6 +662,11 @@ pub async fn run_server( .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) + .nest("/hub", hub_publish::workspaced_service()) + .nest( + "/data_metrics", + windmill_api_workspaces::data_metrics::workspaced_service(), + ) .nest( "/deployment_request", windmill_api_workspaces::deployment_requests::workspaced_service(), diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index d5c759fc18..7c359300c3 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -19,6 +19,30 @@ pub async fn custom_migrations(migrator: &mut CustomMigrator) -> Result<(), Erro tracing::error!("Could not apply flow versioning fix migration: {err:#}"); } + if let Err(err) = normalize_custom_instance_user_attributes(migrator).await { + tracing::error!("Could not normalize custom_instance_user attributes: {err:#}"); + } + + Ok(()) +} + +// Converged on every boot, not once: the one-shot migration swallows errors (it must not +// abort startup without superuser), and an older instance sharing the cluster can re-add +// the attribute. REPLICATION belongs only on custom_instance_replication_user. +async fn normalize_custom_instance_user_attributes( + migrator: &mut CustomMigrator, +) -> Result<(), Error> { + let has_replication = sqlx::query_scalar::<_, bool>( + "SELECT rolreplication FROM pg_roles WHERE rolname = 'custom_instance_user'", + ) + .fetch_optional(migrator.connection()) + .await?; + if has_replication == Some(true) { + sqlx::query("ALTER ROLE custom_instance_user NOREPLICATION") + .execute(migrator.connection()) + .await?; + tracing::info!("Normalized custom_instance_user attributes"); + } Ok(()) } diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index d44484f9db..71becbf9c3 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -26,7 +26,6 @@ pub fn all_tools() -> Vec { ] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -54,7 +53,48 @@ pub fn all_tools() -> Vec { ] })), body_schema: None, - path_field_renames: None, + query_field_renames: None, + body_field_renames: None, + }, + EndpointTool { + name: Cow::Borrowed("listDataMetrics"), + description: Cow::Borrowed("list declared measures and dimensions on DuckLake tables: Call this before writing any aggregate query over a DuckLake table. A declared measure is the canonical definition of that number, and reproducing it yourself will silently disagree with it (a `revenue` measure typically excludes refunds or test rows). Filter by `table` for one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for everything declared under a folder; omit both to browse the whole catalog. Results are keyset-paged: a full page may mean more remain, so continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates can share one GROUP BY. If a number you need has no declared measure, write your own aggregate as usual. Results are limited to declarations whose producing script the caller can read"), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/data_metrics/list"), + method: Cow::Borrowed("GET"), + path_params_schema: None, + query_params_schema: Some(serde_json::json!({ + "type": "object", + "properties": { + "table": { + "type": "string", + "description": "DuckLake table path, with or without the `ducklake://` scheme" + }, + "path_prefix": { + "type": "string", + "description": "Producing script path prefix, e.g. `f/analytics`" + }, + "per_page": { + "type": "integer", + "description": "Results per page, capped at 1000 (default 1000)" + }, + "cursor_table": { + "type": "string", + "description": "Keyset cursor. To page, pass the previous response's `next_cursor` fields back as `cursor_*`; all four move together, and are omitted for the first page. Continue whenever `next_cursor` is present. Every returned row is one the caller may read, so the cursor never names a hidden row.\n" + }, + "cursor_kind": { + "type": "string" + }, + "cursor_name": { + "type": "string" + }, + "cursor_script": { + "type": "string" + } + }, + "required": [] +})), + body_schema: None, query_field_renames: None, body_field_renames: None, }, @@ -124,7 +164,6 @@ pub fn all_tools() -> Vec { "description" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -147,7 +186,6 @@ pub fn all_tools() -> Vec { })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -160,13 +198,12 @@ pub fn all_tools() -> Vec { path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: Some(serde_json::json!({ @@ -205,12 +242,9 @@ pub fn all_tools() -> Vec { }, "path__body": { "type": "string", - "description": "The path to the variable (body parameter)" + "description": "The path to the variable (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -253,7 +287,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -307,7 +340,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -362,7 +394,6 @@ pub fn all_tools() -> Vec { "resource_type" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -385,7 +416,6 @@ pub fn all_tools() -> Vec { })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -398,13 +428,12 @@ pub fn all_tools() -> Vec { path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -433,12 +462,9 @@ pub fn all_tools() -> Vec { }, "path__body": { "type": "string", - "description": "The path to the resource (body parameter)" + "description": "The path to the resource (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -473,7 +499,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -535,7 +560,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -548,7 +572,6 @@ pub fn all_tools() -> Vec { path_params_schema: None, query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -646,7 +669,6 @@ pub fn all_tools() -> Vec { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -699,7 +721,6 @@ Creates a new version of an existing script when called with the same path and t "language" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -722,7 +743,6 @@ Creates a new version of an existing script when called with the same path and t })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -754,7 +774,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -789,7 +808,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -816,7 +834,6 @@ Creates a new version of an existing script when called with the same path and t "description": "The arguments to pass to the script or flow", "additionalProperties": true })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -886,7 +903,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -921,7 +937,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -967,7 +982,6 @@ Creates a new version of an existing script when called with the same path and t ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -980,13 +994,12 @@ Creates a new version of an existing script when called with the same path and t path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -1015,18 +1028,14 @@ Creates a new version of an existing script when called with the same path and t }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } }, "required": [ "summary", - "value", - "path__body" + "value" ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -1061,7 +1070,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1099,7 +1107,6 @@ Creates a new version of an existing script when called with the same path and t "policy" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1112,13 +1119,12 @@ Creates a new version of an existing script when called with the same path and t path_params_schema: Some(serde_json::json!({ "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] })), query_params_schema: None, @@ -1139,12 +1145,9 @@ Creates a new version of an existing script when called with the same path and t }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } } -})), - path_field_renames: Some(serde_json::json!({ - "path__path": "path" })), query_field_renames: None, body_field_renames: Some(serde_json::json!({ @@ -1174,7 +1177,6 @@ Creates a new version of an existing script when called with the same path and t "description": "The arguments to pass to the script or flow", "additionalProperties": true })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1269,7 +1271,6 @@ Creates a new version of an existing script when called with the same path and t "language" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1390,7 +1391,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1557,7 +1557,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1596,7 +1595,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1628,7 +1626,6 @@ Creates a new version of an existing script when called with the same path and t "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -1841,7 +1838,6 @@ You should get the schema of the script or flow before creating the schedule to "args" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2047,7 +2043,6 @@ You should get the schema of the script or flow before updating the schedule to "args" ] })), - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2070,7 +2065,6 @@ You should get the schema of the script or flow before updating the schedule to })), query_params_schema: None, body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2102,7 +2096,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2168,7 +2161,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, }, @@ -2198,7 +2190,6 @@ You should get the schema of the script or flow before updating the schedule to "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, } diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index db1644379d..5d78536d86 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -12,7 +12,7 @@ use windmill_mcp::common::transform::apply_key_transformation; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, WorkspaceInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend}; +use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend, PathFilter}; use crate::auth::AuthCache; use crate::db::ApiAuthed; @@ -78,7 +78,7 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, - path_prefix: Option<&str>, + path_filter: Option>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; get_items::( @@ -87,7 +87,7 @@ impl McpBackend for WindmillBackend { workspace_id, scope_type, "script", - path_prefix, + path_filter, ) .await .map_err(|e| ErrorData::internal_error(e.message, None)) @@ -98,7 +98,7 @@ impl McpBackend for WindmillBackend { auth: &ApiAuthed, workspace_id: &str, favorites_only: bool, - path_prefix: Option<&str>, + path_filter: Option>, ) -> BackendResult> { let scope_type = if favorites_only { "favorites" } else { "all" }; get_items::( @@ -107,7 +107,7 @@ impl McpBackend for WindmillBackend { workspace_id, scope_type, "flow", - path_prefix, + path_filter, ) .await .map_err(|e| ErrorData::internal_error(e.message, None)) @@ -295,7 +295,6 @@ impl McpBackend for WindmillBackend { workspace_id, args_map, &endpoint_tool.path_params_schema, - &endpoint_tool.path_field_renames, )?; let query_string = build_query_string( args_map, diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index de0ad9d4da..d68afe826d 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -15,7 +15,7 @@ use windmill_common::scripts::{get_full_hub_script_by_path, Schema}; use windmill_common::utils::{query_elems_from_hub, StripPath}; use windmill_common::worker::to_raw_value; use windmill_common::{DB, HUB_BASE_URL}; -use windmill_mcp::server::{BackendResult, ErrorData}; +use windmill_mcp::server::{BackendResult, ErrorData, PathFilter}; use windmill_mcp::{HubResponse, HubScriptInfo, ItemSchema, ResourceInfo, ResourceType}; use crate::db::ApiAuthed; @@ -24,6 +24,43 @@ use crate::HTTP_CLIENT; // items max limit const ITEMS_FETCH_MAX_LIMIT: usize = 100; +/// Escape LIKE wildcards so a literal path is matched as a prefix, not a pattern. +fn escape_like(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// Build the SQL condition matching any MCP scope pattern, mirroring +/// `is_resource_allowed`. Returns `None` when no filter should be applied (a `*` +/// pattern grants everything); `Some("false")` when the list is empty (grants +/// nothing); otherwise an OR of per-pattern `o.path` conditions. +fn scope_patterns_condition(patterns: &[String]) -> Option { + if patterns.iter().any(|p| p == "*") { + return None; + } + if patterns.is_empty() { + return Some("false".to_string()); + } + let conds: Vec = patterns + .iter() + .map(|p| { + if let Some(prefix) = p.strip_suffix("/*") { + // A subtree pattern matches the folder itself or anything under it. + let subtree = format!("{}/%", escape_like(prefix)); + format!( + "({} OR {})", + "o.path = ?".bind(&prefix), + "o.path LIKE ? ESCAPE '\\'".bind(&subtree), + ) + } else { + "o.path = ?".bind(p) + } + }) + .collect(); + Some(format!("({})", conds.join(" OR "))) +} + // ============================================================================ // Database utilities // ============================================================================ @@ -135,7 +172,7 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen workspace_id: &str, scope_type: &str, item_type: &str, - path_prefix: Option<&str>, + path_filter: Option>, ) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; @@ -155,12 +192,17 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen sqlb.and_where("(o.auto_kind IS NULL OR o.auto_kind <> 'lib')"); } - if let Some(prefix) = path_prefix { - let escaped = prefix - .replace('\\', "\\\\") - .replace('%', "\\%") - .replace('_', "\\_"); - sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&format!("{}%", escaped))); + match path_filter { + None => {} + Some(PathFilter::Prefix(prefix)) => { + let escaped = format!("{}%", escape_like(prefix)); + sqlb.and_where("o.path LIKE ? ESCAPE '\\'".bind(&escaped)); + } + Some(PathFilter::Patterns(patterns)) => { + if let Some(cond) = scope_patterns_condition(patterns) { + sqlb.and_where(cond); + } + } } sqlb.order_by( @@ -253,7 +295,7 @@ pub async fn get_hub_script_schema(path: &str, db: &DB) -> Result // ============================================================================ /// Look up the original field name from a field_renames map. -/// field_renames maps renamed_key -> original_key (e.g. {"path__path": "path"}). +/// field_renames maps renamed_key -> original_key (e.g. {"path__body": "path"}). fn get_original_name(renamed_key: &str, field_renames: &Option) -> String { field_renames .as_ref() @@ -331,20 +373,17 @@ pub fn substitute_path_params( workspace_id: &str, args_map: &serde_json::Map, path_schema: &Option, - path_field_renames: &Option, ) -> BackendResult { let mut path_template = path.replace("{workspace}", workspace_id); if let Some(schema) = path_schema { if let Some(props) = schema.get("properties").and_then(|p| p.as_object()) { for (param_name, _) in props { - // param_name may be renamed (e.g. "path__path"), get original for URL placeholder - let original_name = get_original_name(param_name, path_field_renames); - let placeholder = format!("{{{}}}", original_name); + let placeholder = format!("{{{}}}", param_name); match args_map.get(param_name) { Some(param_value) => { if let Some(str_val) = param_value.as_str() { - validate_path_param_value(&original_name, str_val)?; + validate_path_param_value(param_name, str_val)?; path_template = path_template.replace(&placeholder, str_val); } } @@ -479,6 +518,42 @@ pub fn build_request_body( } } +/// Scopes to embed in the JWT minted for a proxied MCP endpoint request. The MCP +/// runner already authorized *which* endpoint may be called; this bounds *what +/// the resulting internal request can do*. +/// +/// - Unscoped caller (cookie / full-privilege token): unscoped JWT. +/// - Scope-restricted caller whose own scopes already authorize the route: keep +/// those scopes verbatim, so the target handler's per-path `check_scopes` still +/// enforces the caller's path caps (e.g. a `variables:read:u/admin/safe/*` +/// token can't read `u/admin/secret` via the getVariable proxy). +/// - Otherwise the caller has no route scope for this domain (the common +/// `mcp:`-only token): mint a least-privilege scope for exactly this route, +/// failing closed if the route can't be resolved. +fn jwt_scopes_for_proxied_route( + caller_scopes: Option<&[String]>, + method: &str, + route_path: &str, +) -> BackendResult>> { + let caller_restricted = + caller_scopes.is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:"))); + if !caller_restricted { + return Ok(None); + } + if windmill_api_auth::scopes::check_scopes_for_route(caller_scopes, route_path, method).is_ok() + { + return Ok(caller_scopes.map(|s| s.to_vec())); + } + let scope = + windmill_api_auth::scopes::scope_for_route(method, route_path).ok_or_else(|| { + ErrorData::internal_error( + "Could not derive route scope for proxied MCP endpoint".to_string(), + None, + ) + })?; + Ok(Some(vec![scope])) +} + /// Create HTTP request with authentication pub async fn create_http_request( method: &str, @@ -502,31 +577,12 @@ pub async fn create_http_request( } }; - // Bound the minted JWT to exactly this proxied route so a scope-restricted - // MCP token can't be widened into a full-privilege blank check. The - // endpoint-name gate (in the MCP runner) already authorized *which* endpoint - // may be called; this constrains what the resulting request can do. Unscoped - // callers (cookie / full-privilege tokens) keep an unscoped JWT to preserve - // existing behavior. A scope-restricted caller whose route can't be resolved - // fails closed. - let caller_restricted = api_authed - .scopes - .as_deref() - .is_some_and(|s| s.iter().any(|x| !x.starts_with("if_jobs:filter_tags:"))); - let scopes = if caller_restricted { - let parsed = reqwest::Url::parse(url) - .map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?; - let scope = - windmill_api_auth::scopes::scope_for_route(method, parsed.path()).ok_or_else(|| { - ErrorData::internal_error( - "Could not derive route scope for proxied MCP endpoint".to_string(), - None, - ) - })?; - Some(vec![scope]) - } else { - None - }; + // Scope the minted JWT to the proxied route so a scope-restricted MCP token + // can't be widened into a full-privilege blank check. See + // `jwt_scopes_for_proxied_route`. + let parsed = reqwest::Url::parse(url) + .map_err(|e| ErrorData::internal_error(format!("Invalid proxied URL: {}", e), None))?; + let scopes = jwt_scopes_for_proxied_route(api_authed.scopes.as_deref(), method, parsed.path())?; // Add authorization header let authed = Authed::from(api_authed.clone()); @@ -581,6 +637,59 @@ mod tests { use super::*; use serde_json::json; + fn scopes(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn proxy_jwt_unscoped_caller_keeps_none() { + // No scopes, or filter-tags-only, is treated as unscoped -> unscoped JWT. + assert_eq!( + jwt_scopes_for_proxied_route(None, "GET", "/api/w/ws/variables/get/u/a/b").unwrap(), + None + ); + let ft = scopes(&["if_jobs:filter_tags:foo"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&ft), "GET", "/api/w/ws/variables/get/u/a/b") + .unwrap(), + None + ); + } + + #[test] + fn proxy_jwt_bare_mcp_token_falls_back_to_route_scope() { + // A token whose only authority is its mcp: scope has no variables route + // scope, so the JWT gets a least-privilege route scope for this request. + let s = scopes(&["mcp:endpoints:getVariable"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "GET", "/api/w/ws/variables/get/u/admin/secret") + .unwrap(), + Some(scopes(&["variables:read"])) + ); + } + + #[test] + fn proxy_jwt_mixed_token_passes_through_caller_route_scope() { + // The caller's route scope is preserved so the target handler's per-path + // check_scopes enforces the cap; the coarse route match here is path-blind. + let s = scopes(&["mcp:endpoints:getVariable", "variables:read:u/admin/safe/*"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "GET", "/api/w/ws/variables/get/u/admin/secret") + .unwrap(), + Some(s.clone()) + ); + } + + #[test] + fn proxy_jwt_run_script_bare_mcp_falls_back_to_jobs_run_scripts() { + let s = scopes(&["mcp:scripts:f/team/*", "mcp:endpoints:*"]); + assert_eq!( + jwt_scopes_for_proxied_route(Some(&s), "POST", "/api/w/ws/jobs/run/p/f/team/deploy") + .unwrap(), + Some(scopes(&["jobs:run:scripts"])) + ); + } + #[test] fn build_request_body_passthrough_forwards_script_args_minus_path() { // runScriptByPath-shaped body: additionalProperties, no declared props. @@ -633,6 +742,60 @@ mod tests { ); } + // updateFlow-shaped: `path` is both a path parameter and a body field. The path + // parameter keeps the plain name; only the body side is mangled. + fn update_flow_schemas() -> (Option, Option, Option) { + ( + Some(json!({ + "type": "object", + "properties": { "path": { "type": "string" } }, + "required": ["path"] + })), + Some(json!({ + "type": "object", + "properties": { + "summary": { "type": "string" }, + "value": { "type": "object" }, + "path__body": { "type": "string" } + }, + "required": ["summary", "value"] + })), + Some(json!({ "path__body": "path" })), + ) + } + + #[test] + fn build_request_body_maps_body_path_alias_for_rename() { + // The mangled body field carries the *new* path when renaming; it must reach the + // API under its original name `path`. (An omitted `path__body` is intentionally + // absent from the body; the server defaults it from the URL path parameter.) + let (path_schema, body_schema, body_renames) = update_flow_schemas(); + let args: serde_json::Map = json!({ + "path": "f/team/my_flow", + "path__body": "f/team/renamed_flow", + "summary": "s", + "value": {} + }) + .as_object() + .unwrap() + .clone(); + + let body = build_request_body( + "POST", + &args, + &body_schema, + &body_renames, + &path_schema, + &None, + ) + .expect("body should be built"); + assert_eq!( + body.as_object().unwrap().get("path"), + Some(&json!("f/team/renamed_flow")), + "path__body must be sent as `path` so a rename takes effect" + ); + } + #[test] fn build_request_body_get_has_no_body() { let body_schema = Some(json!({ "type": "object", "additionalProperties": true })); @@ -703,7 +866,6 @@ mod tests { "dev", &args, &path_schema, - &None, ); assert!( result.is_err(), @@ -725,7 +887,6 @@ mod tests { "dev", &args, &path_schema, - &None, ) .expect("legitimate path should substitute"); assert_eq!(result, "/w/dev/scripts/get/p/u/alice/my_script"); @@ -781,4 +942,60 @@ mod tests { "?path=u%2Falice%2Fmy%20script" ); } + + fn strings(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn scope_patterns_condition_wildcard_disables_filter() { + // A `*` pattern grants everything, so no SQL condition should be added. + assert_eq!(scope_patterns_condition(&strings(&["*"])), None); + assert_eq!(scope_patterns_condition(&strings(&["f/team/*", "*"])), None); + } + + #[test] + fn scope_patterns_condition_empty_matches_nothing() { + // An empty pattern list grants no items of this type. + assert_eq!(scope_patterns_condition(&[]), Some("false".to_string())); + } + + #[test] + fn scope_patterns_condition_exact_path() { + assert_eq!( + scope_patterns_condition(&strings(&["u/admin/my_script"])), + Some("(o.path = 'u/admin/my_script')".to_string()) + ); + } + + #[test] + fn scope_patterns_condition_subtree() { + // `f/team/*` matches the folder itself or anything beneath it, mirroring + // resource_matches_pattern. Underscores in the prefix are LIKE-escaped. + assert_eq!( + scope_patterns_condition(&strings(&["f/team/*"])), + Some("((o.path = 'f/team' OR o.path LIKE 'f/team/%' ESCAPE '\\'))".to_string()) + ); + } + + #[test] + fn scope_patterns_condition_mixed_ored() { + assert_eq!( + scope_patterns_condition(&strings(&["u/admin/one", "f/team/*"])), + Some( + "(o.path = 'u/admin/one' OR (o.path = 'f/team' OR o.path LIKE 'f/team/%' ESCAPE '\\'))" + .to_string() + ) + ); + } + + #[test] + fn scope_patterns_condition_escapes_like_wildcards() { + // A subtree prefix containing `%`/`_` must be escaped so it isn't treated + // as a LIKE pattern; the exact-match arm is quoted verbatim by bind. + assert_eq!( + scope_patterns_condition(&strings(&["f/a_b/*"])), + Some("((o.path = 'f/a_b' OR o.path LIKE 'f/a\\_b/%' ESCAPE '\\'))".to_string()) + ); + } } diff --git a/backend/windmill-api/src/offboarding.rs b/backend/windmill-api/src/offboarding.rs index 24b6d3f10d..f9bf0e1449 100644 --- a/backend/windmill-api/src/offboarding.rs +++ b/backend/windmill-api/src/offboarding.rs @@ -186,6 +186,7 @@ async fn get_offboard_preview( "kafka_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "nats_trigger", "sqs_trigger", "gcp_trigger", @@ -750,6 +751,7 @@ async fn check_path_conflicts( "kafka_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "nats_trigger", "sqs_trigger", "gcp_trigger", @@ -1017,13 +1019,14 @@ async fn offboard_user_from_workspace<'c>( &new_permissioned_as, username, w_id ).execute(&mut **tx).await?; - // ---- triggers (all 9 types with path/permissioned_as) ---- + // ---- triggers (all 10 types with path/permissioned_as) ---- let trigger_tables = [ "http_trigger", "websocket_trigger", "kafka_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "nats_trigger", "sqs_trigger", "gcp_trigger", diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs new file mode 100644 index 0000000000..489c0d5dd5 --- /dev/null +++ b/backend/windmill-api/src/runnables.rs @@ -0,0 +1,541 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Unified, keyset-paginated listing of a workspace's runnables (scripts, +//! flows, apps) merged into one globally-ordered stream. The homepage uses it +//! so a chosen order (recently updated / oldest / name) is correct and complete +//! across all three kinds at any workspace size, instead of client-sorting a +//! per-kind capped window. +//! +//! Efficiency: each kind is a UNION ALL branch ordered by an index on +//! `(workspace_id, archived, )` (created_at / edited_at, or the lowered +//! summary-or-path expression for name orders); Postgres merges the ordered +//! branches and stops at the page limit. Pagination is keyset — a +//! `(sort_key, path, kind, tiebreak)` cursor, where `tiebreak` (a script's hash, +//! 0 for flow/app) is a stable final key that keeps the order total even when rows +//! tie on (sort_key, path, kind) — so deep pages don't re-scan. Visibility is +//! enforced in-SQL by RLS via the `user_db` transaction. + +use crate::db::{ApiAuthed, DB}; +use crate::utils::{build_scope_path_filter, ScopePathFilter}; +use axum::{ + extract::{Extension, Path, Query}, + routing::get, + Json, Router, +}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use serde::{Deserialize, Serialize}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult}, +}; +use windmill_types::scripts::ScriptHash; +use windmill_types::user_drafts::DraftUserRef; + +pub fn workspaced_service() -> Router { + Router::new().route("/list", get(list_runnables)) +} + +#[derive(Deserialize)] +struct ListRunnablesQuery { + /// `updated` (default) or `name`. + order_by: Option, + /// Descending when true (default true). + order_desc: Option, + /// Comma-separated subset of `script,flow,app`; omitted means all. + kinds: Option, + show_archived: Option, + /// Include library scripts (no runnable main). Ignored for flows/apps. + include_without_main: Option, + /// Restrict to paths under this prefix (owner/folder filter). + path_start: Option, + /// Comma-separated labels; a row matches if it (or its folder) carries all. + label: Option, + /// Case-insensitive substring match on summary or path. + search: Option, + per_page: Option, + /// Opaque keyset cursor from a previous page's `next_cursor`. + cursor: Option, +} + +// Absent optional fields are omitted (not serialized as null) to match the +// per-kind list contract; the frontend row components expect `undefined`. +#[derive(Serialize, sqlx::FromRow)] +struct RunnableItem { + #[serde(rename = "type")] + kind: String, // 'script' | 'flow' | 'app' + path: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + workspace_id: String, + extra_perms: serde_json::Value, + starred: bool, + archived: bool, + is_draft: bool, + #[serde(skip_serializing_if = "Option::is_none")] + draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + draft_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + draft_users: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + inherited_labels: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + ws_error_handler_muted: Option, + #[serde(skip_serializing_if = "Option::is_none")] + edited_at: Option>, + // script-only. ScriptHash serializes as the 16-char hex string that + // /scripts/get/{hash} parses (a raw i64 would produce a broken link). + #[serde(skip_serializing_if = "Option::is_none")] + hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + language: Option, + #[serde(rename = "kind", skip_serializing_if = "Option::is_none")] + script_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auto_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + use_codebase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + has_deploy_errors: Option, + // app-only + #[serde(skip_serializing_if = "Option::is_none")] + raw_app: Option, + #[serde(skip_serializing_if = "Option::is_none")] + execution_mode: Option, + #[serde(skip_serializing_if = "Option::is_none")] + id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + version: Option, + // sort keys, echoed into the cursor (not serialized to the client) + #[serde(skip)] + sort_time: chrono::DateTime, + #[serde(skip)] + sort_name: String, + // Final tiebreaker making the sort total: a script's hash, 0 for flow/app. A stable + // last key so rows that tie on (sort_key, path, kind) still have a strict order and + // none is skipped when a tie crosses a page boundary. + #[serde(skip)] + tiebreak: i64, +} + +#[derive(Serialize)] +struct ListRunnablesResponse { + items: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + next_cursor: Option, +} + +#[derive(Serialize, Deserialize)] +struct Cursor { + /// sort key of the last row: rfc3339 timestamp (updated) or lowered name. + k: String, + p: String, + t: String, + /// tiebreak (script hash / 0) of the last row. + #[serde(default)] + tb: i64, +} + +fn encode_cursor(item: &RunnableItem, order_by_name: bool) -> String { + let k = if order_by_name { + item.sort_name.clone() + } else { + item.sort_time.to_rfc3339() + }; + let c = Cursor { k, p: item.path.clone(), t: item.kind.clone(), tb: item.tiebreak }; + URL_SAFE_NO_PAD.encode(serde_json::to_vec(&c).unwrap_or_default()) +} + +fn decode_cursor(raw: &str) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(raw) + .map_err(|_| Error::BadRequest("invalid cursor".to_string()))?; + serde_json::from_slice(&bytes).map_err(|_| Error::BadRequest("invalid cursor".to_string())) +} + +/// Escape LIKE/ILIKE wildcards so a caller value (search term, path/scope +/// prefix) matches literally. Relies on the default `\` escape character. +fn escape_like(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// The three UNION-ALL branch SELECTs, each projecting the shared `RunnableItem` +/// column set (NULL for columns that don't apply to that kind). `$1`=workspace, +/// `$2`=username (favorites), `$3`=email (drafts). Kind-specific and per-request +/// WHERE fragments are appended by the caller. +struct Branches { + script: String, + flow: String, + app: String, +} + +fn branch_sqls() -> Branches { + // draft_users subquery (correlated) mirrors the per-kind list endpoints; run + // only for the returned page, so its cost is bounded by per_page. + let draft_users = |typ_pred: &str| -> String { + format!( + "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ + FROM draft d \ + LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ + WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND {typ_pred}) as draft_users" + ) + }; + + let script = format!( + "SELECT 'script' as kind, o.path, o.summary, o.workspace_id, o.extra_perms, \ + favorite.path IS NOT NULL as starred, o.archived, \ + draft.email IS NOT NULL as is_draft, NULL::bool as draft_only, NULL::text as draft_path, \ + {draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \ + o.ws_error_handler_muted, o.created_at as edited_at, \ + o.hash, o.language::text as language, o.kind::text as script_kind, o.auto_kind, \ + o.codebase IS NOT NULL as use_codebase, \ + (o.lock_error_logs IS NOT NULL) as has_deploy_errors, \ + NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + o.created_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, o.hash as tiebreak \ + FROM script o \ + LEFT JOIN favorite ON favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ + LEFT JOIN draft ON draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script' AND draft.email = $3", + draft_users = draft_users("d.typ = 'script'") + ); + + let flow = format!( + "SELECT 'flow' as kind, o.path, o.summary, o.workspace_id, o.extra_perms, \ + favorite.path IS NOT NULL as starred, o.archived, \ + draft.email IS NOT NULL as is_draft, NULL::bool as draft_only, NULL::text as draft_path, \ + {draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \ + o.ws_error_handler_muted, o.edited_at, \ + NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ + NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ + NULL::bool as raw_app, NULL::text as execution_mode, NULL::bigint as id, NULL::bigint as version, \ + o.edited_at as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ + FROM flow o \ + LEFT JOIN favorite ON favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ + LEFT JOIN draft ON draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow' AND draft.email = $3", + draft_users = draft_users("d.typ = 'flow'") + ); + + let app = format!( + "SELECT 'app' as kind, o.path, o.summary, o.workspace_id, o.extra_perms, \ + favorite.path IS NOT NULL as starred, false as archived, \ + draft.path IS NOT NULL as is_draft, NULL::bool as draft_only, NULL::text as draft_path, \ + {draft_users}, o.labels, folder_labels(o.workspace_id, o.path) as inherited_labels, \ + NULL::bool as ws_error_handler_muted, av.created_at as edited_at, \ + NULL::bigint as hash, NULL::text as language, NULL::text as script_kind, NULL::text as auto_kind, \ + NULL::bool as use_codebase, NULL::bool as has_deploy_errors, \ + av.raw_app, o.policy->>'execution_mode' as execution_mode, o.id, \ + o.versions[array_upper(o.versions, 1)] as version, \ + COALESCE(av.created_at, 'epoch'::timestamptz) as sort_time, lower(COALESCE(NULLIF(o.summary, ''), o.path)) as sort_name, 0::bigint as tiebreak \ + FROM app o \ + LEFT JOIN favorite ON favorite.favorite_kind = 'app' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = $2 \ + LEFT JOIN (SELECT DISTINCT path, workspace_id FROM draft WHERE typ IN ('app', 'raw_app') AND email = $3) draft ON draft.path = o.path AND draft.workspace_id = o.workspace_id \ + LEFT JOIN app_version av ON av.id = o.versions[array_upper(o.versions, 1)]", + draft_users = draft_users("d.typ IN ('app', 'raw_app')") + ); + + Branches { script, flow, app } +} + +async fn list_runnables( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(_db): Extension, + Path(w_id): Path, + Query(q): Query, +) -> JsonResult { + let order_by_name = q.order_by.as_deref() == Some("name"); + let desc = q.order_desc.unwrap_or(true); + let per_page = q.per_page.unwrap_or(50).clamp(1, 1000); + let show_archived = q.show_archived.unwrap_or(false); + let order_dir = if desc { "DESC" } else { "ASC" }; + let sort_col = if order_by_name { + "sort_name" + } else { + "sort_time" + }; + + let mut kinds: Vec<&str> = match q.kinds.as_deref() { + None | Some("") => vec!["script", "flow", "app"], + Some(csv) => csv + .split(',') + .map(|s| s.trim()) + .filter(|s| ["script", "flow", "app"].contains(s)) + .collect(), + }; + // Apps carry no `archived` column and are never listed as archived. + if show_archived { + kinds.retain(|k| *k != "app"); + } + // Operators may only see scripts. + if authed.is_operator { + kinds.retain(|k| *k == "script"); + } + + let branches = branch_sqls(); + + // Params after the fixed $1=w_id, $2=username, $3=email. `add_bind` returns + // the next placeholder (`$N`) and records the value in order. + let mut binds: Vec = vec![]; + let add_bind = |binds: &mut Vec, v: String| -> String { + binds.push(v); + format!("${}", 3 + binds.len()) + }; + + let mut common: Vec = vec!["o.workspace_id = $1".to_string()]; + if let Some(ps) = q.path_start.as_ref().filter(|s| !s.is_empty()) { + let p = add_bind(&mut binds, format!("{}%", escape_like(ps))); + common.push(format!("o.path LIKE {}", p)); + } + if let Some(search) = q.search.as_ref().filter(|s| !s.is_empty()) { + let p = add_bind(&mut binds, format!("%{}%", escape_like(search))); + common.push(format!("(o.summary ILIKE {p} OR o.path ILIKE {p})")); + } + if let Some(label) = q.label.as_ref().filter(|s| !s.is_empty()) { + for l in label.split(',') { + let p = add_bind(&mut binds, l.trim().to_string()); + common.push(format!( + "(o.labels @> ARRAY[{p}] OR folder_labels(o.workspace_id, o.path) @> ARRAY[{p}])" + )); + } + } + let common_where = common.join(" AND "); + + // Keyset predicate for pages after the first (non-starred rows only). A + // row-value comparison keeps the composite order; the key is cast to the + // branch column's type. + let keyset_sql: Option = match &q.cursor { + Some(raw) => { + let cur = decode_cursor(raw)?; + let kp = add_bind(&mut binds, cur.k); + let pp = add_bind(&mut binds, cur.p); + let tp = add_bind(&mut binds, cur.t); + let tbp = add_bind(&mut binds, cur.tb.to_string()); + let cmp = if desc { "<" } else { ">" }; + let key_cast = if order_by_name { + format!("{}::text", kp) + } else { + format!("{}::timestamptz", kp) + }; + Some(format!( + "({sort_col}, path, kind, tiebreak) {cmp} ({key_cast}, {pp}::text, {tp}::text, {tbp}::bigint)" + )) + } + None => None, + }; + + // Fine-grained scoped tokens (e.g. `scripts:read:f/foo/*`) must be confined to + // their granted paths. RLS alone doesn't honor token scopes, so push the + // per-domain path grant into SQL (empty grant -> the branch matches nothing). + // Unscoped sessions -> AllowAll -> no predicate. + let scope_where = |filter: ScopePathFilter, binds: &mut Vec| -> Option { + match filter { + ScopePathFilter::AllowAll => None, + ScopePathFilter::Restricted { exact, prefix } => { + let mut terms: Vec = vec![]; + for e in exact { + binds.push(e); + terms.push(format!("o.path = ${}", 3 + binds.len())); + } + for pre in prefix { + binds.push(pre.clone()); + let pe = format!("${}", 3 + binds.len()); + binds.push(format!("{}/%", escape_like(&pre))); + let pl = format!("${}", 3 + binds.len()); + terms.push(format!("(o.path = {} OR o.path LIKE {})", pe, pl)); + } + Some(if terms.is_empty() { + "false".to_string() + } else { + format!("({})", terms.join(" OR ")) + }) + } + } + }; + // Only push scope binds for kinds whose branch is actually included: a scoped token + // with e.g. `kinds=script` omits the flow/app branches, so binding their scope values + // (which no SQL references) would make the parameter count mismatch and 500. + let script_scope = if kinds.contains(&"script") { + scope_where( + build_scope_path_filter(&authed, "scripts", "read"), + &mut binds, + ) + } else { + None + }; + let flow_scope = if kinds.contains(&"flow") { + scope_where( + build_scope_path_filter(&authed, "flows", "read"), + &mut binds, + ) + } else { + None + }; + let app_scope = if kinds.contains(&"app") { + scope_where(build_scope_path_filter(&authed, "apps", "read"), &mut binds) + } else { + None + }; + + // Per-kind archived predicate (scripts/flows have the column; apps don't and + // are excluded from the archived view). + let archived_pred = if show_archived { + "o.archived = true" + } else { + "o.archived = false" + }; + let mut script_extras: Vec = vec![]; + if !q.include_without_main.unwrap_or(false) || authed.is_operator { + script_extras.push("(o.auto_kind IS NULL OR o.auto_kind <> 'lib')".to_string()); + } + script_extras.push(archived_pred.to_string()); + if show_archived { + // The script table keeps every version as its own row and marks superseded + // ones archived=true, so a bare `archived = true` would surface an active + // path's old versions and repeat a genuinely archived path once per version. + // Match the canonical script listing: only a path whose LATEST row is archived + // belongs in the archived view. (Flows/apps are one row per path, so this only + // applies to scripts.) + script_extras.push( + "o.ctid = (SELECT ctid FROM script s2 WHERE s2.path = o.path \ + AND s2.workspace_id = o.workspace_id ORDER BY s2.created_at DESC LIMIT 1)" + .to_string(), + ); + } + if let Some(s) = &script_scope { + script_extras.push(s.clone()); + } + let mut flow_extras: Vec = vec![archived_pred.to_string()]; + if let Some(s) = &flow_scope { + flow_extras.push(s.clone()); + } + let mut app_extras: Vec = vec![]; + if let Some(s) = &app_scope { + app_extras.push(s.clone()); + } + + // Favorite filter for a branch: Some(true) = starred only, Some(false) = + // non-starred only, None = no filter. Both views pin starred on the first page + // (each is one row per path), so the paged stream always passes Some(false). + let build_branch = |base: &str, + kind: &str, + extras: &[String], + fav: Option, + keyset: Option<&str>, + limit: Option| + -> String { + // Base-table predicates go inside the projection subquery (they read + // o.*/favorite.*); the keyset reads the projected sort aliases, so it + // sits in the wrapper WHERE where those aliases are visible. + let mut w = vec![common_where.clone()]; + w.extend(extras.iter().cloned()); + match fav { + Some(true) => w.push("favorite.path IS NOT NULL".to_string()), + Some(false) => w.push("favorite.path IS NULL".to_string()), + None => {} + } + let keyset_clause = keyset + .map(|ks| format!(" WHERE {}", ks)) + .unwrap_or_default(); + // Per-branch LIMIT so each branch's correlated projections (draft_users, + // folder_labels) are evaluated only for its own top rows, not the whole + // table; the outer union re-limits to the global page. + let limit_clause = limit.map(|n| format!(" LIMIT {}", n)).unwrap_or_default(); + format!( + "(SELECT * FROM ({base} WHERE {where_}) {kind}_b{keyset_clause} ORDER BY {sort_col} {dir}, path {dir}, kind {dir}, tiebreak {dir}{limit_clause})", + where_ = w.join(" AND "), + dir = order_dir, + ) + }; + + let branch_for = |kind: &str, + fav: Option, + keyset: Option<&str>, + limit: Option| + -> Option { + if !kinds.contains(&kind) { + return None; + } + let (base, extras): (&str, &[String]) = match kind { + "script" => (&branches.script, &script_extras), + "flow" => (&branches.flow, &flow_extras), + "app" => (&branches.app, &app_extras), + _ => return None, + }; + Some(build_branch(base, kind, extras, fav, keyset, limit)) + }; + + let run_union = |branches_sql: Vec, limit: Option| -> String { + let unioned = branches_sql.join(" UNION ALL "); + let limit_clause = limit.map(|n| format!(" LIMIT {}", n)).unwrap_or_default(); + format!( + "SELECT * FROM ({unioned}) q ORDER BY {sort_col} {dir}, path {dir}, kind {dir}, tiebreak {dir}{limit_clause}", + dir = order_dir, + ) + }; + + let mut tx = user_db.begin(&authed).await?; + let mut items: Vec = vec![]; + let first_page = q.cursor.is_none(); + + // Pin starred on the first page. Both views are now one row per path (the + // archived view filters to each path's latest row, see archived_pred), so a + // favorite is a single row in either — the starred-first contract holds in the + // archived view too, and the pinned first page stays bounded. + if first_page { + let starred_branches: Vec = ["script", "flow", "app"] + .iter() + .filter_map(|k| branch_for(k, Some(true), None, None)) + .collect(); + if !starred_branches.is_empty() { + let sql = run_union(starred_branches, None); + let mut query = sqlx::query_as::<_, RunnableItem>(&sql) + .bind(&w_id) + .bind(&authed.username) + .bind(&authed.email); + for b in &binds { + query = query.bind(b); + } + items.extend(query.fetch_all(&mut *tx).await?); + } + } + + // Main paged stream: non-starred rows (starred were pinned on the first page above). + let main_fav = Some(false); + let ns_branches: Vec = ["script", "flow", "app"] + .iter() + .filter_map(|k| branch_for(k, main_fav, keyset_sql.as_deref(), Some(per_page))) + .collect(); + + let mut next_cursor: Option = None; + if !ns_branches.is_empty() { + let sql = run_union(ns_branches, Some(per_page)); + let mut query = sqlx::query_as::<_, RunnableItem>(&sql) + .bind(&w_id) + .bind(&authed.username) + .bind(&authed.email); + for b in &binds { + query = query.bind(b); + } + let ns = query.fetch_all(&mut *tx).await?; + if ns.len() == per_page { + if let Some(last) = ns.last() { + next_cursor = Some(encode_cursor(last, order_by_name)); + } + } + items.extend(ns); + } + + tx.commit().await?; + + Ok(Json(ListRunnablesResponse { items, next_cursor })) +} diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index 7a4a4887f8..c7229fe056 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -24,6 +24,7 @@ fn build_trigger_scope_domains() -> Vec { ("kafka_triggers", "Kafka"), ("nats_triggers", "NATS"), ("mqtt_triggers", "MQTT"), + ("amqp_triggers", "AMQP"), ("sqs_triggers", "AWS SQS"), ("gcp_triggers", "GCP Pub/Sub"), ("azure_triggers", "Azure Event Grid"), @@ -207,6 +208,21 @@ lazy_static! { }], }); + // Read-only: the `data_metrics/list` route is the only surface and the + // catalog is written at deploy, never through a token. Path-selectable + // because the route filters rows by the caller's `data_metrics:read` path + // grants. Its own domain, not a `scripts` alias, so a metrics token can't + // reach `/scripts` routes. + groups.push(ScopeDomain { + name: "Data Metrics".to_string(), + description: Some("Read-only access to declared measures and dimensions".to_string()), + scopes: vec![ScopeOption { + value: "data_metrics:read".to_string(), + label: "Read".to_string(), + requires_resource_path: true, + }], + }); + groups.extend(build_standard_scope_domains()); groups.extend(build_trigger_scope_domains()); @@ -235,7 +251,33 @@ mod tests { .flat_map(|d| d.scopes.iter()) .map(|s| s.value.as_str()) .collect(); - assert!(values.contains(&"docs:read"), "docs:read must be selectable"); + assert!( + values.contains(&"docs:read"), + "docs:read must be selectable" + ); assert!(!values.contains(&"docs:write"), "docs has no write surface"); } + + /// The `data_metrics` route enforces its own scope domain, so `data_metrics:read` + /// must be grantable here or no token can ever reach it. It is read-only (the + /// catalog is written at deploy) and path-selectable. + #[test] + fn data_metrics_read_scope_is_exposed_read_only_and_path_selectable() { + let opt = ALL_SCOPES + .iter() + .flat_map(|d| d.scopes.iter()) + .find(|s| s.value == "data_metrics:read") + .expect("data_metrics:read must be selectable"); + assert!( + opt.requires_resource_path, + "data_metrics:read is path-scoped" + ); + assert!( + !ALL_SCOPES + .iter() + .flat_map(|d| d.scopes.iter()) + .any(|s| s.value == "data_metrics:write"), + "data_metrics has no write surface" + ); + } } diff --git a/backend/windmill-api/src/trash.rs b/backend/windmill-api/src/trash.rs index d0d7ddc6c2..416546b850 100644 --- a/backend/windmill-api/src/trash.rs +++ b/backend/windmill-api/src/trash.rs @@ -467,6 +467,7 @@ async fn restore_trigger(tx: &mut sqlx::PgConnection, item: &TrashItemWithData) "nats_trigger", "postgres_trigger", "mqtt_trigger", + "amqp_trigger", "sqs_trigger", "gcp_trigger", "azure_trigger", diff --git a/backend/windmill-api/src/triggers/amqp/mod.rs b/backend/windmill-api/src/triggers/amqp/mod.rs new file mode 100644 index 0000000000..81aaca3aa8 --- /dev/null +++ b/backend/windmill-api/src/triggers/amqp/mod.rs @@ -0,0 +1 @@ +pub use windmill_trigger_amqp::*; diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs index 35b025965d..d86912d8d0 100644 --- a/backend/windmill-api/src/triggers/handler.rs +++ b/backend/windmill-api/src/triggers/handler.rs @@ -61,6 +61,16 @@ pub fn generate_trigger_routers() -> Router { ); } + #[cfg(feature = "amqp_trigger")] + { + use crate::triggers::amqp::AmqpTrigger; + + router = router.nest( + AmqpTrigger::ROUTE_PREFIX, + complete_trigger_routes(AmqpTrigger), + ); + } + #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] { use crate::triggers::sqs::SqsTrigger; @@ -143,6 +153,7 @@ pub struct TriggersCount { nats_count: i64, postgres_count: i64, mqtt_count: i64, + amqp_count: i64, sqs_count: i64, gcp_count: i64, azure_count: i64, @@ -245,6 +256,17 @@ pub async fn get_triggers_count_internal( #[cfg(not(feature = "mqtt_trigger"))] let mqtt_count = 0; + #[cfg(feature = "amqp_trigger")] + let amqp_count = { + use crate::triggers::amqp::AmqpTrigger; + let count = AmqpTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "amqp_trigger"))] + let amqp_count = 0; + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] let sqs_count = { use crate::triggers::sqs::SqsTrigger; @@ -362,6 +384,7 @@ pub async fn get_triggers_count_internal( nats_count, postgres_count, mqtt_count, + amqp_count, gcp_count, azure_count, sqs_count, diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index 43cabf13e3..d12ff7a1e2 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -51,6 +51,14 @@ pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receive listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx) } + #[cfg(feature = "amqp_trigger")] + { + let amqp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::amqp::AmqpTrigger; + + listen_to(AmqpTrigger, db.clone(), amqp_killpill_rx) + } + #[cfg(feature = "websocket")] { let mqtt_killpill_rx = killpill_rx.resubscribe(); diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index bc69c66b2b..10f45cbd7f 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -1,4 +1,6 @@ // Concrete trigger submodules (feature-gated) +#[cfg(feature = "amqp_trigger")] +pub mod amqp; #[cfg(all(feature = "azure_trigger", feature = "enterprise", feature = "private"))] pub mod azure; #[cfg(all(feature = "smtp", feature = "private"))] diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 5f472c49d3..3f27247d23 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -10,7 +10,8 @@ use axum::{body::Body, response::Response}; use serde::{Deserialize, Deserializer}; pub use windmill_api_auth::{ - build_scope_path_predicate, check_scopes, require_devops_role, require_super_admin, + build_scope_path_filter, build_scope_path_predicate, check_scopes, require_devops_role, + require_super_admin, ScopePathFilter, }; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 839057b4b9..c869d87780 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -19,6 +19,7 @@ use windmill_api_auth::check_scopes; feature = "websocket", feature = "postgres_trigger", feature = "mqtt_trigger", + feature = "amqp_trigger", all( feature = "enterprise", any( @@ -144,6 +145,7 @@ pub fn is_none_or_false(val: &Option) -> bool { feature = "websocket", feature = "postgres_trigger", feature = "mqtt_trigger", + feature = "amqp_trigger", feature = "native_trigger", all( feature = "enterprise", @@ -189,6 +191,7 @@ async fn fork_parent_trigger_modes( feature = "websocket", feature = "postgres_trigger", feature = "mqtt_trigger", + feature = "amqp_trigger", feature = "native_trigger", all( feature = "enterprise", @@ -1226,6 +1229,36 @@ pub(crate) async fn tarball_workspace( } } + #[cfg(feature = "amqp_trigger")] + { + use crate::triggers::amqp::AmqpTrigger; + let handler = AmqpTrigger; + let amqp_triggers = handler.list_triggers(&mut *tx, &w_id, None, None).await?; + let parent_modes = fork_parent_trigger_modes( + &db, + ::TABLE_NAME, + parent_workspace_id.as_deref(), + ) + .await?; + + for trigger in amqp_triggers { + let mode_override = trigger_mode_override(&parent_modes, &trigger.base.path); + let trigger_str = &to_string_without_metadata_inner( + &trigger, + ExtraPermsBehavior::Drop, + None, + mode_override.as_ref(), + ) + .unwrap(); + archive + .write_to_archive( + &trigger_str, + &format!("{}.amqp_trigger.json", trigger.base.path), + ) + .await?; + } + } + #[cfg(all(feature = "enterprise", feature = "smtp", feature = "private"))] { use crate::triggers::email::EmailTrigger; @@ -1422,6 +1455,35 @@ pub(crate) async fn tarball_workspace( .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility + // Server-owned auto-pull state (the HMAC webhook secret + hook id/error and + // the synced-sha / last-pull status) must never leave the server: keep it out + // of export archives and synced repos, and don't let a re-imported workspace + // inherit another install's hook/sync state. Mirrors the GET-settings redaction. + fn redact_git_sync_for_export(git_sync: Option) -> Option { + let mut git_sync = git_sync?; + if let Some(repos) = git_sync + .get_mut("repositories") + .and_then(|r| r.as_array_mut()) + { + for repo in repos { + if let Some(auto_pull) = + repo.get_mut("auto_pull").and_then(|a| a.as_object_mut()) + { + for field in [ + "webhook_secret", + "webhook_id", + "webhook_error", + "last_synced_sha", + "last_pull_status", + ] { + auto_pull.remove(field); + } + } + } + } + Some(git_sync) + } + let settings_str = if settings_version.as_deref() == Some("v2") { let settings = SimplifiedSettings { auto_invite: row.auto_invite, @@ -1431,7 +1493,7 @@ pub(crate) async fn tarball_workspace( success_handler: row.success_handler, ai_config: row.ai_config, large_file_storage: row.large_file_storage, - git_sync: row.git_sync, + git_sync: redact_git_sync_for_export(row.git_sync), default_app: row.default_app, default_scripts: row.default_scripts, name: row.name.clone().unwrap_or_default(), @@ -1502,7 +1564,7 @@ pub(crate) async fn tarball_workspace( error_handler_muted_on_cancel, ai_config: row.ai_config, large_file_storage: row.large_file_storage, - git_sync: row.git_sync, + git_sync: redact_git_sync_for_export(row.git_sync), default_app: row.default_app, default_scripts: row.default_scripts, name: row.name.unwrap_or_default(), diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 7e30547d9b..e45bba05c3 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -38,6 +38,7 @@ anyhow.workspace = true serde.workspace = true serde_json.workspace = true serde_yml.workspace = true +memchr.workspace = true erased-serde = "0.4" chrono.workspace = true chrono-tz.workspace = true @@ -90,6 +91,7 @@ const-str.workspace = true crc.workspace = true windmill-macros.workspace = true windmill-parser-sql.workspace = true +windmill-parser-sql-asset.workspace = true windmill-parser-ts.workspace = true windmill-parser-py = { workspace = true, optional = true } windmill-parser.workspace = true diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 62f6d32070..d6a4156f38 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -567,11 +567,9 @@ mod trigger_ref_roundtrip_tests { // `trigger_spec_to_row` rebuilds a stored ref as `s3://`, and // `parse_asset_trigger_ref` parses it back. The two must be inverse for // every S3 URI form, or a consumer's `// on` trigger lands on a different - // graph node than the producer's inferred write. Because `parse_asset_syntax` - // strips ALL leading slashes, a canonical path never starts with `/`, so the - // naive `prefix + path` rebuild round-trips — including the `S3Object(s3="/x")` - // quad-slash case that previously desynced (path `/x` rebuilt to `s3:///x`, - // which re-parsed to `x`). + // graph node than the producer's inferred write. `parse_asset_syntax` + // keeps the URI suffix verbatim (a default-storage path starts with `/`), + // so the naive `prefix + path` rebuild round-trips for every form. fn roundtrip(uri: &str) -> String { let (pkind, path) = parse_asset_syntax(uri, false).expect("parse uri"); assert_eq!(pkind, PAssetKind::S3Object); @@ -590,11 +588,10 @@ mod trigger_ref_roundtrip_tests { #[test] fn s3_trigger_ref_roundtrips_for_every_uri_form() { - assert_eq!(roundtrip("s3:///exports/x"), "exports/x"); // SDK default storage - assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // DuckDB / bare + assert_eq!(roundtrip("s3:///exports/x"), "/exports/x"); // SDK default storage + assert_eq!(roundtrip("s3://exports/x"), "exports/x"); // named storage `exports` assert_eq!(roundtrip("s3://mybucket/exports/x"), "mybucket/exports/x"); // explicit - assert_eq!(roundtrip("s3:////x"), "x"); // S3Object(s3="/x") quad-slash - assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "y=2024/f.parquet"); // Hive + assert_eq!(roundtrip("s3:///y=2024/f.parquet"), "/y=2024/f.parquet"); // Hive } } @@ -632,6 +629,7 @@ pub fn trigger_spec_to_row(spec: &TriggerSpec) -> Option<(ScriptTriggerKind, Str | TriggerSpec::Email | TriggerSpec::Kafka | TriggerSpec::Mqtt + | TriggerSpec::Amqp | TriggerSpec::Nats | TriggerSpec::Postgres | TriggerSpec::Sqs diff --git a/backend/windmill-common/src/data_metrics.rs b/backend/windmill-common/src/data_metrics.rs new file mode 100644 index 0000000000..41eaeae9c3 --- /dev/null +++ b/backend/windmill-common/src/data_metrics.rs @@ -0,0 +1,268 @@ +//! Catalog of table-scoped metric declarations (`// measure`, `// dimension`). +//! +//! Declarations live in the producing script's annotation header and are mirrored +//! into the `data_metric` table on deploy, so a reader can ask "what does this +//! table declare?" or "what is declared under this folder?" without fetching and +//! parsing script bodies. Nothing here rewrites or executes SQL: the catalog is +//! read by the script editor and by agents, which compose their own queries. + +use crate::error::{Error, Result}; +use serde::Serialize; +use windmill_parser::asset_parser::{parse_pipeline_annotations, AssetKind}; + +pub const KIND_MEASURE: &str = "measure"; +pub const KIND_DIMENSION: &str = "dimension"; + +// Mirror the `data_metric` column widths so oversized annotations fail at deploy +// with a clear message instead of a Postgres error. +const MAX_NAME_LEN: usize = 255; +const MAX_TABLE_PATH_LEN: usize = 510; + +/// Canonical `/.
`, defaulting the schema to DuckLake's `main`. +/// +/// A producer's `// materialize ducklake://lake/orders` target omits the schema +/// while a consumer reading `lake.main.orders` records its asset as +/// `lake/main.orders`. Both sides must land on the same catalog key or a consumer +/// can never resolve the table's declarations. +pub fn canonical_table_path(path: &str) -> String { + let p = path.strip_prefix("ducklake://").unwrap_or(path); + match p.split_once('/') { + Some((lake, rest)) if !rest.contains('.') => format!("{lake}/main.{rest}"), + _ => p.to_string(), + } +} + +/// Whether a canonical table path is safe to interpolate into an `ATTACH` string. +/// A real DuckLake path is dotted/slashed identifiers; anything else could break +/// out of the string literal or the statement. +pub fn is_safe_table_path(path: &str) -> bool { + !path.is_empty() + && path + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.' | '/')) +} + +/// One declared measure or dimension. +#[derive(Serialize, Debug, Clone, PartialEq)] +pub struct MetricEntry { + /// The declaring script, and the path reads are authorized against. + pub script_path: String, + /// Canonical scheme-less ducklake path, `/.
`. + pub table_path: String, + pub kind: String, + pub name: String, + pub expr: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub filter: Option, +} + +/// Replace `script_path`'s catalog rows with the declarations currently in its +/// source, so the catalog always describes the deployed state. +/// +/// Declarations describe the table the script materializes, so a script without a +/// ducklake `// materialize` target contributes nothing: there would be no table +/// to attach them to. +/// +/// **Authorization is the caller's responsibility.** This writes workspace-scoped +/// rows for whatever `w_id`/`script_path` it is handed and performs no permission +/// check of its own. Call it only where the caller has already established write +/// access to that script path, as script deployment does. +pub async fn sync_metric_catalog( + db: &mut sqlx::PgConnection, + w_id: &str, + script_path: &str, + old_path: Option<&str>, + content: &str, +) -> Result<()> { + // Clear the old path too, so a rename does not orphan its declarations at a + // path this script no longer occupies. + sqlx::query!( + "DELETE FROM data_metric WHERE workspace_id = $1 AND (script_path = $2 OR script_path = $3)", + w_id, + script_path, + old_path.unwrap_or(script_path) + ) + .execute(&mut *db) + .await?; + + let ann = parse_pipeline_annotations(content); + let Some(spec) = ann + .materialize + .filter(|m| m.target_kind == AssetKind::Ducklake) + else { + return Ok(()); + }; + if ann.measures.is_empty() && ann.dimensions.is_empty() { + return Ok(()); + } + + // The lake/table name is interpolated into an `ATTACH 'ducklake://' …` + // string that a *reader* executes, so a quote or semicolon in it is stored SQL + // injection. + let canonical = canonical_table_path(&spec.target_path); + if !is_safe_table_path(&canonical) { + return Err(Error::BadRequest(format!( + "`// materialize` target `{}` has an unsafe table path `{canonical}`: a metric's \ + lake/table name may contain only letters, digits, `_ - . /`", + spec.target_path + ))); + } + + // Reject caller-authored values longer than their columns (VARCHAR counts + // characters) with a clear error at deploy, not a Postgres "value too long" + // failure. script_path is the script's own already-validated path. + if canonical.chars().count() > MAX_TABLE_PATH_LEN { + return Err(Error::BadRequest(format!( + "`// materialize` target table path `{canonical}` is too long (max {MAX_TABLE_PATH_LEN} characters)" + ))); + } + for name in ann + .measures + .iter() + .map(|m| &m.name) + .chain(ann.dimensions.iter().map(|d| &d.name)) + { + if name.chars().count() > MAX_NAME_LEN { + return Err(Error::BadRequest(format!( + "metric name `{name}` is too long (max {MAX_NAME_LEN} characters)" + ))); + } + } + + // Declarations are interpolated verbatim into SQL that a *reader* executes, so + // each one must be a single expression. Otherwise `count(*) FROM t; DELETE …` + // stored as a measure would run as whoever opens the drawer. + for (kind, name, body) in ann + .measures + .iter() + .flat_map(|m| { + std::iter::once((KIND_MEASURE, &m.name, m.expr.as_str())) + .chain(m.filter.as_deref().map(|f| (KIND_MEASURE, &m.name, f))) + }) + .chain( + ann.dimensions + .iter() + .map(|d| (KIND_DIMENSION, &d.name, d.expr.as_str())), + ) + { + if !windmill_parser_sql_asset::is_single_sql_expression(body) { + return Err(Error::BadRequest(format!( + "{kind} `{name}` must be a single SQL expression; `{body}` is not \ + (it would be executed by anyone reading this table's metrics)" + ))); + } + } + + // A measure's trailing `where` compiles to ` FILTER (WHERE )`, and + // SQL binds FILTER to one aggregate call. A composite like `sum(a)/count(b)` + // would filter only `count(b)`, silently corrupting the canonical number, so a + // filtered measure must be a single aggregate call. + for m in ann.measures.iter().filter(|m| m.filter.is_some()) { + if !windmill_parser_sql_asset::is_single_function_call(&m.expr) { + return Err(Error::BadRequest(format!( + "measure `{}` has a `where` filter, so `{}` must be a single aggregate call \ + (e.g. `sum(amount)`): a `where` on a composite like `sum(a)/count(b)` would \ + filter only part of it and silently produce the wrong number", + m.name, m.expr + ))); + } + } + + let mut kinds: Vec = Vec::new(); + let mut names: Vec = Vec::new(); + let mut exprs: Vec = Vec::new(); + let mut filters: Vec> = Vec::new(); + for m in &ann.measures { + kinds.push(KIND_MEASURE.to_string()); + names.push(m.name.clone()); + exprs.push(m.expr.clone()); + filters.push(m.filter.clone()); + } + for d in &ann.dimensions { + kinds.push(KIND_DIMENSION.to_string()); + names.push(d.name.clone()); + exprs.push(d.expr.clone()); + filters.push(None); + } + + sqlx::query!( + "INSERT INTO data_metric (workspace_id, script_path, table_path, kind, name, expr, filter) \ + SELECT $1, $2, $3, k, n, e, f \ + FROM UNNEST($4::text[], $5::text[], $6::text[], $7::text[]) AS t(k, n, e, f)", + w_id, + script_path, + &canonical, + &kinds[..], + &names[..], + &exprs[..], + &filters[..] as &[Option] + ) + .execute(&mut *db) + .await?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_declaration_smuggling_extra_statements_is_rejected() { + use windmill_parser_sql_asset::is_single_sql_expression; + // The shape that would otherwise execute as whoever opens the drawer. + assert!(!is_single_sql_expression( + "count(*) FROM t; DELETE FROM secrets; SELECT count(*)" + )); + assert!(!is_single_sql_expression("1; DROP TABLE t")); + assert!(!is_single_sql_expression("sum(amount) garbage")); + // A trailing comment is skipped by the parser, so it is rejected explicitly. + assert!(!is_single_sql_expression("sum(amount) --rest")); + assert!(!is_single_sql_expression("sum(amount) /* c */")); + // Ordinary declarations still pass. + assert!(is_single_sql_expression("sum(amount)")); + assert!(is_single_sql_expression("count(*)")); + assert!(is_single_sql_expression("not is_refund")); + assert!(is_single_sql_expression("date_trunc('month', ordered_at)")); + } + + #[test] + fn a_filtered_measure_must_be_a_single_aggregate_call() { + use windmill_parser_sql_asset::is_single_function_call; + // A filtered composite would mis-apply FILTER to only one of its aggregates. + assert!(!is_single_function_call("sum(amount) / count(*)")); + assert!(!is_single_function_call("sum(a) + sum(b)")); + assert!(!is_single_function_call("sum(amount) * 2")); + // A single aggregate call is fine (FILTER attaches unambiguously). + assert!(is_single_function_call("sum(amount)")); + assert!(is_single_function_call("count(*)")); + assert!(is_single_function_call("count(distinct user_id)")); + } + + #[test] + fn an_unsafe_lake_path_is_rejected() { + // The injection shape: a quote/semicolon in the lake name would break out of + // the ATTACH string literal. + assert!(!is_safe_table_path("dl';SELECT(1);--/main.orders")); + assert!(!is_safe_table_path("lake\"/main.t")); + assert!(!is_safe_table_path("")); + // Ordinary canonical paths are fine. + assert!(is_safe_table_path("sales/main.orders")); + assert!(is_safe_table_path("my-lake/analytics.daily_v2")); + } + + #[test] + fn a_producers_target_and_a_consumers_read_canonicalize_alike() { + // What a producer declares, and what a consumer's read records. + assert_eq!( + canonical_table_path("ducklake://sales/orders"), + canonical_table_path("sales/main.orders") + ); + assert_eq!(canonical_table_path("sales/orders"), "sales/main.orders"); + // An explicit schema is preserved rather than forced to `main`. + assert_eq!( + canonical_table_path("sales/analytics.orders"), + "sales/analytics.orders" + ); + } +} diff --git a/backend/windmill-common/src/ee_oss.rs b/backend/windmill-common/src/ee_oss.rs index 26b9c87b7e..2510df0960 100644 --- a/backend/windmill-common/src/ee_oss.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -83,6 +83,11 @@ pub async fn enforce_offline_caps(_db: &DB) -> anyhow::Result, + /// Comma-separated host/IP patterns for which the MITM proxy skips upstream TLS + /// verification. Unlike `no_proxy_hosts` the hosts stay traced — only the proxy's own + /// upstream certificate check is disabled. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub insecure_upstream_hosts: Option, + /// Extra CA certificates (PEM bundle) added to the MITM proxy's upstream trust store, + /// on top of the system roots, so internal endpoints signed by a private CA verify. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub upstream_ca_certs: Option, } /// Script language identifier (for instance config use). @@ -767,7 +776,10 @@ pub enum DucklakeCatalogResourceType { // Custom instance PG databases // --------------------------------------------------------------------------- -/// Custom PostgreSQL databases managed by the instance. +/// Custom PostgreSQL databases managed by the instance. `user_pwd` is operator-configurable +/// (resolved from a Kubernetes secretKeyRef by the EE operator); `databases` is runtime +/// setup status. The replication-role password lives in a separate hidden setting +/// (`custom_instance_replication_pwd`), never in this operator-facing config row. #[derive(Deserialize, Serialize, Clone, Debug, Default)] #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] pub struct CustomInstancePgDatabases { @@ -936,6 +948,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_user_pg_pwd", "ducklake_settings", "custom_instance_pg_databases", + "custom_instance_replication_pwd", "uid", "rsa_keys", "jwt_secret", @@ -957,6 +970,10 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ // every bulk InstanceSettings save via `GlobalSettings::extra`. Hiding it // on read + rejecting it in `diff_global_settings` breaks that loop. "worker_configs", + // Auto-generated password for the REPLICATION role used by postgres triggers. + // Server-only (written by setup/refresh via direct SQL), never operator-authored — + // hidden so the config machinery can't read, rewrite, or drop it. + "custom_instance_replication_pwd", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -967,6 +984,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[ "hub_api_secret", "license_key", "ducklake_user_pg_pwd", + "custom_instance_replication_pwd", "pip_index_url", "pip_extra_index_url", "npm_config_registry", @@ -991,6 +1009,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ "object_store_cache_config", &["secret_key", "serviceAccountKey"], ), + ("custom_instance_pg_databases", &["user_pwd"]), ]; fn redact_json_value(value: &serde_json::Value) -> serde_json::Value { @@ -1015,10 +1034,7 @@ fn mask_nested_sensitive(key: &str, value: &serde_json::Value) -> serde_json::Va } } // Settings that are maps-of-objects where each child has a sensitive sub-field. - const NESTED_MAP_SENSITIVE: &[(&str, &str)] = &[ - ("oauths", "secret"), - ("custom_instance_pg_databases", "user_pwd"), - ]; + const NESTED_MAP_SENSITIVE: &[(&str, &str)] = &[("oauths", "secret")]; for &(parent_key, child_field) in NESTED_MAP_SENSITIVE { if key == parent_key { if let serde_json::Value::Object(entries) = value { @@ -1133,14 +1149,13 @@ pub fn diff_global_settings( let mut previous_values = BTreeMap::new(); let mut unchanged_count: usize = 0; for (key, desired_value) in desired { - // `worker_configs` is a legacy ghost: worker configs belong in the - // `config` table with a `worker__` prefix. If a client PUT carries a - // top-level `worker_configs` key (it flattens into - // `GlobalSettings::extra` on deserialize), drop it here instead of - // letting it resurrect a stale `global_settings` row. - if key == "worker_configs" { + // Hidden settings are server-managed and never driven by config: they are + // filtered out on read (`from_db`) and must be ignored on write too, so a client + // PUT that flattened one into `GlobalSettings::extra` can't resurrect or clobber + // the row (e.g. `worker_configs`, or the custom-instance credentials/status). + if HIDDEN_SETTINGS.contains(&key.as_str()) { tracing::warn!( - "Ignoring 'worker_configs' in global_settings diff: worker configs must be written to the config table (worker__ prefix), not global_settings" + "Ignoring hidden setting '{key}' in global_settings diff (server-managed, not configurable)" ); continue; } @@ -2358,34 +2373,63 @@ mod tests { } #[test] - fn custom_instance_pg_databases_roundtrips() { + fn custom_instance_replication_pwd_is_isolated_from_config() { + // The replication-role password is server-only: written by setup/refresh via direct + // SQL, never operator-authored. It must stay out of the declarative config surface + // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it. + assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd")); + assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd")); + assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd")); + + // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted. + let mut desired = BTreeMap::new(); + desired.insert( + "custom_instance_replication_pwd".to_string(), + serde_json::json!("attacker-set"), + ); + let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge); + assert!( + diff.upserts.is_empty(), + "hidden setting must not be upserted" + ); + + // A current value is never deleted by a Replace that omits it. + let mut current = BTreeMap::new(); + current.insert( + "custom_instance_replication_pwd".to_string(), + serde_json::json!("live"), + ); + let diff = diff_global_settings(¤t, &BTreeMap::new(), ApplyMode::Replace); + assert!( + !diff + .deletes + .contains(&"custom_instance_replication_pwd".to_string()), + "hidden setting must not be deleted" + ); + } + + #[test] + fn custom_instance_pg_databases_roundtrips_and_redacts_user_pwd() { + // user_pwd stays operator-configurable (EE secretKeyRef); databases is runtime status. let json = r#"{ "user_pwd": "secret123", - "databases": { - "mydb": { - "logs": { - "super_admin": "OK", - "database_credentials": "OK", - "valid_dbname": "OK", - "created_database": "OK", - "db_connect": "OK", - "grant_permissions": "OK" - }, - "success": true, - "tag": "production" - } - } + "databases": { "mydb": { "success": true, "tag": "production" } } }"#; let pg: CustomInstancePgDatabases = serde_json::from_str(json).unwrap(); assert_eq!( pg.user_pwd.as_ref().and_then(|v| v.as_literal()), Some("secret123") ); - let db = &pg.databases["mydb"]; - assert!(db.success); - assert_eq!(db.tag.as_deref(), Some("production")); - assert_eq!(db.logs.super_admin, "OK"); - assert_eq!(db.logs.grant_permissions, "OK"); + assert!(pg.databases["mydb"].success); + + let out = format_setting_value( + "custom_instance_pg_databases", + &serde_json::json!({ "user_pwd": "user-plaintext-password" }), + ); + assert!( + !out.contains("user-plaintext-password"), + "user_pwd leaked: {out}" + ); } #[test] diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 2178d9a023..d73637026a 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -20,6 +20,7 @@ use crate::{ users::username_to_permissioned_as, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, + workspaces::workspace_with_fork_ancestors, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -343,8 +344,8 @@ lazy_static::lazy_static! { ).unwrap_or(false); } -pub async fn check_tag_available_for_workspace_internal<'c>( - db: impl sqlx::PgExecutor<'c>, +pub async fn check_tag_available_for_workspace_internal( + db: &DB, w_id: &str, tag: &str, email: &str, @@ -361,7 +362,14 @@ pub async fn check_tag_available_for_workspace_internal<'c>( if custom_tags_per_w.global.contains(&tag.to_string()) { is_tag_in_workspace_custom_tags = true; } else if let Some(specific_tag) = custom_tags_per_w.specific.get(tag) { - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(w_id); + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if specific_tag.is_fork_scoped() { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); } match is_tag_in_scope_tags { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 8f3dae3203..08a4ea93ed 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -62,6 +62,7 @@ pub mod instance_config; pub mod job_metrics; pub mod log_context; pub mod materialization; +pub mod data_metrics; pub mod min_version; pub mod notify_events; pub mod runtime_assets; diff --git a/backend/windmill-common/src/min_version.rs b/backend/windmill-common/src/min_version.rs index 2b1a218f23..45d356bb44 100644 --- a/backend/windmill-common/src/min_version.rs +++ b/backend/windmill-common/src/min_version.rs @@ -203,7 +203,7 @@ pub async fn update_min_version( Connection::Sql(db) => { for worker_name in &_worker_names { crate::ee::simple_alert_helper( - format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately."), + async { format!("Worker {worker_name} version {current} is below minimum keep-alive version {min_keep_alive}. Upgrade immediately.") }, format!("Worker {worker_name} version {current} is now at or above minimum keep-alive version {min_keep_alive}."), &format!("worker-below-min-keep-alive-{worker_name}"), || current < min_keep_alive, diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index ce7dbc2533..063d3958de 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -4835,6 +4835,10 @@ fn pg_action_to_string(action: &str) -> String { pub async fn pg_get_full_schema( client: &tokio_postgres::Client, ) -> Result { + // Primary-key and default-value info are joined in (a table has at most one + // primary-key constraint, so `pkc` stays 1:1) rather than fetched via + // per-column correlated subqueries — on large catalogs those subqueries run + // once per column and make the introspection time out. let column_rows = client .query( "SELECT @@ -4842,19 +4846,17 @@ pub async fn pg_get_full_schema( c.relname AS table_name, a.attname AS column_name, pg_catalog.format_type(a.atttypid, a.atttypmod) AS datatype, - (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128) - FROM pg_catalog.pg_attrdef d - WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default_value, - CASE a.attnotnull WHEN false THEN true ELSE false END AS nullable, - EXISTS ( - SELECT 1 FROM pg_catalog.pg_index i - WHERE i.indrelid = c.oid AND i.indisprimary AND a.attnum = ANY(i.indkey) - ) AS is_primary_key, - (SELECT con.conname FROM pg_catalog.pg_constraint con - WHERE con.conrelid = c.oid AND con.contype = 'p' LIMIT 1) AS pk_constraint_name + substring(pg_catalog.pg_get_expr(ad.adbin, ad.adrelid, true) for 128) AS default_value, + NOT a.attnotnull AS nullable, + COALESCE(pkc.conkey @> ARRAY[a.attnum], false) AS is_primary_key, + pkc.conname AS pk_constraint_name FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON a.attrelid = c.oid JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid + LEFT JOIN pg_catalog.pg_attrdef ad + ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum AND a.atthasdef + LEFT JOIN pg_catalog.pg_constraint pkc + ON pkc.conrelid = c.oid AND pkc.contype = 'p' WHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped diff --git a/backend/windmill-common/src/schema_contracts.rs b/backend/windmill-common/src/schema_contracts.rs index b605aae33b..7d941040bc 100644 --- a/backend/windmill-common/src/schema_contracts.rs +++ b/backend/windmill-common/src/schema_contracts.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use sqlx::types::Json; use sqlx::{Postgres, Transaction}; use windmill_parser::asset_parser::{ - ColumnLineage, DataTest, MaterializeSpec, OnSchemaChange, PARTITION_TOKEN, + ColumnLineage, DataTest, Dimension, MaterializeSpec, Measure, OnSchemaChange, PARTITION_TOKEN, }; use windmill_types::assets::{AssetKind, AssetWithAltAccessType}; @@ -46,6 +46,15 @@ pub enum ContractWarningKind { /// Relationship join columns have different captured types (may still /// coerce at run time — phrased as "differs", not "will fail"). RelationshipTypeMismatch, + /// A `// measure` body reads a column absent from the producer's own + /// captured (target) schema. + MissingMeasureColumn, + /// A `// dimension` body reads a column absent from the producer's own + /// captured (target) schema. + MissingDimensionColumn, + /// A `// measure` body contains no aggregate (it is a row-level expression), + /// so grouping it by a dimension produces an invalid query. + NonAggregateMeasure, /// Warnings for this asset were suppressed by the producer's /// `on_schema_change=ignore` (one informational entry per asset). Suppressed, @@ -299,6 +308,109 @@ pub fn diff_contract( warnings } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetricRefKind { + Measure, + Dimension, +} + +/// Warn about measures that contain no aggregate. A measure is meant to reduce +/// the table's rows, so grouping a row-level expression like `amount` by a +/// dimension yields an invalid query. Pure and schema-independent. A warning, not +/// an error: detection is deliberately lenient (any function call is accepted, to +/// avoid a reserved aggregate-name list), so a hard block would risk rejecting an +/// unusual-but-valid aggregate. +pub fn check_measures_aggregate( + measures: &[Measure], + materialize: Option<&MaterializeSpec>, +) -> Vec { + let mut warnings = vec![]; + let target = materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake) + .map(|m| normalize_asset_path(&m.target_path)) + .unwrap_or_default(); + for m in measures { + if windmill_parser_sql_asset::measure_expr_may_aggregate(&m.expr) { + continue; + } + warnings.push(ContractWarning { + kind: ContractWarningKind::NonAggregateMeasure, + asset_path: target.clone(), + column: None, + expected_type: None, + found_type: None, + schema_version: None, + captured_at: None, + message: format!( + "`// measure {} = {}` contains no aggregate, so grouping it by a \ + dimension produces an invalid query", + m.name, m.expr + ), + }); + } + warnings +} + +/// A metric measure/dimension plus the column names its body reads, extracted by +/// the async wrapper (the pure diff never parses SQL). One per declared metric. +#[derive(Debug, Clone)] +pub struct MetricColumnRef { + pub kind: MetricRefKind, + pub name: String, + pub columns: Vec, +} + +/// Validate metric measure/dimension bodies against the producer's own captured +/// (target) schema. Separate from `diff_contract` because these are the +/// producer's *own* declarations, not consumer reads of an upstream: a measure +/// citing a column the producer itself dropped is the producer's bug, and is +/// deliberately not muted by `on_schema_change=ignore` (which only governs +/// downstream drift). Pure — column refs are pre-extracted by the caller. +pub fn diff_metric_contract( + metric_refs: &[MetricColumnRef], + materialize: Option<&MaterializeSpec>, + schemas: &HashMap, +) -> Vec { + let mut warnings: Vec = vec![]; + let Some(m) = + materialize.filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake) + else { + return warnings; + }; + let own_path = normalize_asset_path(&m.target_path); + let Some(schema) = schemas.get(&own_path) else { + return warnings; + }; + for mr in metric_refs { + for col in &mr.columns { + if is_reserved(col) || schema.find(col).is_some() { + continue; + } + let (kind, label) = match mr.kind { + MetricRefKind::Measure => (ContractWarningKind::MissingMeasureColumn, "measure"), + MetricRefKind::Dimension => { + (ContractWarningKind::MissingDimensionColumn, "dimension") + } + }; + warnings.push(ContractWarning { + kind, + asset_path: own_path.clone(), + column: Some(col.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// {label} {}` reads `{col}`, which is not in ducklake://{own_path}'s \ + captured schema (v{})", + mr.name, schema.version + ), + }); + } + } + warnings +} + /// Load captured schemas + producer modes and run the contract check for one /// consumer script's parsed refs. /// @@ -313,7 +425,13 @@ pub async fn check_schema_contracts( column_lineage: &[ColumnLineage], data_tests: &[DataTest], materialize: Option<&MaterializeSpec>, + measures: &[Measure], + dimensions: &[Dimension], ) -> Result> { + // Whether a measure aggregates is a pure property of its expression, so this + // check runs regardless of whether a schema has been captured. + let mut warnings = check_measures_aggregate(measures, materialize); + // Referenced ducklake paths (normalized) across every ref family the diff // inspects — plus the consumer's own materialize target (for W3 types). let mut paths: HashSet = HashSet::new(); @@ -342,7 +460,9 @@ pub async fn check_schema_contracts( } } if paths.is_empty() { - return Ok(vec![]); + // No captured schemas to diff against, but the schema-independent measure + // checks above still stand. + return Ok(warnings); } // A managed scd2 producer (re)creates a `_current` view with the base @@ -462,14 +582,41 @@ pub async fn check_schema_contracts( } } - Ok(diff_contract( + // Extract the columns each measure/dimension body reads (measure filters + // too), then validate them against the producer's own captured schema. + let mut metric_refs: Vec = + Vec::with_capacity(measures.len() + dimensions.len()); + for mm in measures { + let mut columns = windmill_parser_sql_asset::extract_expr_column_idents(&mm.expr); + if let Some(filter) = &mm.filter { + columns.extend(windmill_parser_sql_asset::extract_expr_column_idents( + filter, + )); + } + metric_refs.push(MetricColumnRef { + kind: MetricRefKind::Measure, + name: mm.name.clone(), + columns, + }); + } + for dd in dimensions { + metric_refs.push(MetricColumnRef { + kind: MetricRefKind::Dimension, + name: dd.name.clone(), + columns: windmill_parser_sql_asset::extract_expr_column_idents(&dd.expr), + }); + } + + warnings.extend(diff_contract( assets, column_lineage, data_tests, materialize, &schemas, &ignored, - )) + )); + warnings.extend(diff_metric_contract(&metric_refs, materialize, &schemas)); + Ok(warnings) } #[cfg(test)] @@ -531,6 +678,65 @@ mod tests { assert!(diff_contract(&[a], &[], &[], None, &schemas, &HashSet::new()).is_empty()); } + #[test] + fn metric_column_missing_from_own_schema_warns() { + let ann = parse_pipeline_annotations("-- materialize ducklake://lake/orders\nSELECT 1;"); + let schemas = HashMap::from([( + "lake/orders".to_string(), + schema(&[("amount", "DOUBLE"), ("region", "VARCHAR")]), + )]); + let refs = vec![ + MetricColumnRef { + kind: MetricRefKind::Measure, + name: "revenue".to_string(), + columns: vec!["amount".to_string()], + }, + MetricColumnRef { + kind: MetricRefKind::Measure, + name: "refunds".to_string(), + columns: vec!["refund_amt".to_string()], + }, + MetricColumnRef { + kind: MetricRefKind::Dimension, + name: "zone".to_string(), + columns: vec!["zone_id".to_string()], + }, + ]; + let w = diff_metric_contract(&refs, ann.materialize.as_ref(), &schemas); + // `amount` exists; the refund measure and zone dimension cite unknown + // columns of the producer's own captured schema. + assert_eq!(w.len(), 2); + assert_eq!(w[0].kind, ContractWarningKind::MissingMeasureColumn); + assert_eq!(w[0].column.as_deref(), Some("refund_amt")); + assert_eq!(w[1].kind, ContractWarningKind::MissingDimensionColumn); + assert_eq!(w[1].column.as_deref(), Some("zone_id")); + // No captured schema → silent (first deploy). + assert!(diff_metric_contract(&refs, ann.materialize.as_ref(), &HashMap::new()).is_empty()); + } + + #[test] + fn a_measure_with_no_aggregate_warns_schema_independently() { + let ann = parse_pipeline_annotations( + "-- materialize ducklake://sales/orders\n\ + -- measure revenue = amount\n\ + -- measure scaled = amount * 2\n\ + -- measure total = sum(amount)\n\ + -- measure n = count(*)\n\ + -- measure custom = my_udaf(x)\n\ + SELECT 1;", + ); + // Runs without any captured schema. Bare column and pure arithmetic warn; + // sum/count and an unknown function (benefit of the doubt, no aggregate-name + // list) do not. + let w = check_measures_aggregate(&ann.measures, ann.materialize.as_ref()); + assert_eq!(w.len(), 2); + assert!(w + .iter() + .all(|w| w.kind == ContractWarningKind::NonAggregateMeasure)); + assert!(w[0].message.contains("revenue")); + assert!(w[1].message.contains("scaled")); + } + #[test] fn asset_without_captured_schema_is_silent() { let assets = vec![read_asset("lake/unknown", &["whatever"])]; diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 627636b8f5..99471acfe9 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -1,4 +1,4 @@ -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use crate::error::Error; @@ -58,6 +58,63 @@ impl std::fmt::Display for SsrfValidationError { // `anyhow::Result` (e.g. the EE SAML metadata loader). impl std::error::Error for SsrfValidationError {} +/// A URL that passed SSRF validation, carrying the exact addresses its host +/// resolved to so the eventual connect targets the SAME address that was +/// checked. +/// +/// Validation resolves the host once and verifies every address is public; it +/// then hands those addresses back instead of discarding them. Callers pin them +/// onto their client — [`apply_dns_pinning`](ValidatedTarget::apply_dns_pinning) +/// for reqwest, or [`pinned_addrs`](ValidatedTarget::pinned_addrs) for a raw TCP +/// connect — so a DNS rebinder cannot answer a public IP at check-time and an +/// internal one (e.g. 169.254.169.254) at connect-time. Without pinning the +/// check and the connect resolve independently and the guard is a TOCTOU no-op. +/// +/// `addrs` is empty when the host was an IP literal (there is nothing to rebind) +/// or when an `ALLOW_PRIVATE_*` override skipped resolution entirely; pinning is +/// then a no-op and the caller connects normally. +/// +/// Limitation: pinning governs only *direct* connections. When a deployment +/// configures an outbound egress proxy (`HTTP_PROXY`/`HTTPS_PROXY`), the proxy +/// resolves the target host itself and the pin does not reach it — a property of +/// proxy-based egress shared by every app-side SSRF guard, not specific to this +/// one. The public/private pre-check still runs; closing the proxy hop would +/// require the proxy to resolve, which it owns. +#[derive(Debug, Clone)] +pub struct ValidatedTarget { + /// The URL host, exactly as reqwest keys its DNS override on. + pub host: String, + /// Public addresses the host resolved to, to pin at connect time. + pub addrs: Vec, +} + +impl ValidatedTarget { + /// A target with nothing to pin: an IP-literal host (no rebinding possible) + /// or a host whose SSRF check was skipped by an `ALLOW_PRIVATE_*` override. + fn unpinned(host: &str) -> Self { + ValidatedTarget { host: host.to_string(), addrs: Vec::new() } + } + + /// Pin the validated addresses onto a reqwest client builder so connect-time + /// resolution cannot diverge from what was checked. No-op when there is + /// nothing to pin (IP-literal host, or a skipped `ALLOW_PRIVATE_*` check). + pub fn apply_dns_pinning(&self, builder: reqwest::ClientBuilder) -> reqwest::ClientBuilder { + if self.addrs.is_empty() { + builder + } else { + builder.resolve_to_addrs(&self.host, &self.addrs) + } + } + + /// The validated addresses to connect to, for callers that pin by opening + /// the socket themselves (e.g. the WebSocket trigger's raw TCP connect) + /// rather than through reqwest. Empty means "nothing to pin, connect + /// normally". + pub fn pinned_addrs(&self) -> &[SocketAddr] { + &self.addrs + } +} + impl From for Error { fn from(e: SsrfValidationError) -> Self { Error::BadRequest(e.to_string()) @@ -69,8 +126,13 @@ impl From for Error { /// Checks: /// 1. Scheme must be http or https /// 2. Host must be present and not a private/loopback/link-local IP -/// 3. DNS resolution is checked to prevent DNS rebinding to internal IPs -pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError> { +/// 3. The host is resolved and every address verified public +/// +/// Returns the resolved addresses as a [`ValidatedTarget`] so the caller can pin +/// them onto the client that actually connects. Validating here and re-resolving +/// at connect time is a TOCTOU no-op against a DNS rebinder — the check only +/// closes the hole if the connect targets the SAME address this resolved. +pub async fn validate_url_for_ssrf(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -85,12 +147,13 @@ pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError> // 2. Host check let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; - // 3. If the host is an IP literal, check it directly + // 3. If the host is an IP literal, check it directly. There is nothing to + // rebind (reqwest connects straight to the literal), so no addresses to pin. if let Ok(ip) = host.parse::() { if is_private_ip(&ip) { return Err(SsrfValidationError::Private { resolved: false }); } - return Ok(()); + return Ok(ValidatedTarget::unpinned(host)); } // 4. DNS resolution check — resolve the hostname and verify all IPs are public @@ -117,7 +180,7 @@ pub async fn validate_url_for_ssrf(url: &str) -> Result<(), SsrfValidationError> } } - Ok(()) + Ok(ValidatedTarget { host: host.to_string(), addrs }) } pub fn allow_private_mcp_server_urls() -> bool { @@ -132,7 +195,7 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } -pub async fn validate_saml_metadata_url(url: &str) -> Result<(), SsrfValidationError> { +pub async fn validate_saml_metadata_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -141,16 +204,16 @@ pub async fn validate_saml_metadata_url(url: &str) -> Result<(), SsrfValidationE scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), } - parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; if allow_private_saml_metadata_urls() { - return Ok(()); + return Ok(ValidatedTarget::unpinned(host)); } validate_url_for_ssrf(url).await } -pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationError> { +pub async fn validate_mcp_server_url(url: &str) -> Result { let parsed = url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; @@ -159,16 +222,24 @@ pub async fn validate_mcp_server_url(url: &str) -> Result<(), SsrfValidationErro scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), } - parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; if allow_private_mcp_server_urls() { - return Ok(()); + return Ok(ValidatedTarget::unpinned(host)); } validate_url_for_ssrf(url).await } -pub async fn validate_mcp_server_url_for_bad_request(url: &str, label: &str) -> Result<(), Error> { +/// Validate an MCP-related URL and return the [`ValidatedTarget`] so the caller +/// can pin the connect: the OAuth registration/discovery/token requests carry +/// secrets, so they must target the validated address (see +/// `windmill_mcp::oauth::no_redirect_http_client_pinned`). Callers that only +/// pre-validate (no adjacent connect) can discard the target. +pub async fn validate_mcp_server_url_for_bad_request( + url: &str, + label: &str, +) -> Result { validate_mcp_server_url(url).await.map_err(|e| { Error::BadRequest(format!( "{label} is not allowed: {}", @@ -331,6 +402,35 @@ mod tests { assert!(validate_url_for_ssrf("https://google.com").await.is_ok()); } + /// An IP-literal host has nothing to rebind — reqwest connects straight to + /// the literal — so the target pins no addresses. + #[tokio::test] + async fn validate_url_ip_literal_pins_nothing() { + let target = validate_url_for_ssrf("http://8.8.8.8:1234/x") + .await + .unwrap(); + assert_eq!(target.host, "8.8.8.8"); + assert!(target.pinned_addrs().is_empty()); + } + + /// Regression for the DNS-rebinding TOCTOU: the guard must surface the exact + /// public addresses it validated so the caller can pin the connect to the + /// SAME address. If + /// this returned nothing, the connect would re-resolve and a rebinder could + /// swap in an internal IP after the check. + #[tokio::test] + async fn validate_url_surfaces_resolved_addrs_for_pinning() { + let target = validate_url_for_ssrf("https://google.com").await.unwrap(); + assert_eq!(target.host, "google.com"); + assert!(!target.pinned_addrs().is_empty()); + assert!(target + .pinned_addrs() + .iter() + .all(|a| !is_private_ip(&a.ip()))); + // The pin applies cleanly onto a reqwest builder. + let _ = target.apply_dns_pinning(reqwest::ClientBuilder::new()); + } + /// Regression for #9171: a malformed base URL (missing scheme) must report /// `InvalidUrl`/`DisallowedScheme`, not `Private` — only `Private` gets the /// "set ALLOW_PRIVATE_AI_BASE_URLS" hint, which is misleading for a typo'd diff --git a/backend/windmill-common/src/triggers.rs b/backend/windmill-common/src/triggers.rs index b4f8f472cc..d667363ebb 100644 --- a/backend/windmill-common/src/triggers.rs +++ b/backend/windmill-common/src/triggers.rs @@ -39,7 +39,8 @@ pub async fn update_triggers_script_path( t3 AS (UPDATE postgres_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), \ t4 AS (UPDATE mqtt_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), \ t5 AS (UPDATE nats_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), \ - t6 AS (UPDATE sqs_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4) \ + t6 AS (UPDATE sqs_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4), \ + t7 AS (UPDATE amqp_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4) \ UPDATE gcp_trigger SET script_path = $1, server_id = NULL WHERE script_path = $2 AND workspace_id = $3 AND is_flow = $4", new_path, old_path, diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 7e0c1f3f55..84a0e8c3ea 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -45,6 +45,7 @@ pub enum UserDraftItemKind { TriggerKafka, TriggerNats, TriggerMqtt, + TriggerAmqp, TriggerSqs, TriggerGcp, TriggerAzure, @@ -80,6 +81,7 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerKafka => "trigger_kafka", UserDraftItemKind::TriggerNats => "trigger_nats", UserDraftItemKind::TriggerMqtt => "trigger_mqtt", + UserDraftItemKind::TriggerAmqp => "trigger_amqp", UserDraftItemKind::TriggerSqs => "trigger_sqs", UserDraftItemKind::TriggerGcp => "trigger_gcp", UserDraftItemKind::TriggerAzure => "trigger_azure", @@ -94,7 +96,7 @@ impl UserDraftItemKind { /// Every variant, for code that must enumerate kinds (e.g. generating /// the `draft_only` existence SQL). - pub const ALL: [UserDraftItemKind; 25] = [ + pub const ALL: [UserDraftItemKind; 26] = [ UserDraftItemKind::Script, UserDraftItemKind::Flow, UserDraftItemKind::App, @@ -111,6 +113,7 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerKafka, UserDraftItemKind::TriggerNats, UserDraftItemKind::TriggerMqtt, + UserDraftItemKind::TriggerAmqp, UserDraftItemKind::TriggerSqs, UserDraftItemKind::TriggerGcp, UserDraftItemKind::TriggerAzure, @@ -144,6 +147,7 @@ impl UserDraftItemKind { TriggerKafka => Some("kafka_trigger"), TriggerNats => Some("nats_trigger"), TriggerMqtt => Some("mqtt_trigger"), + TriggerAmqp => Some("amqp_trigger"), TriggerSqs => Some("sqs_trigger"), TriggerGcp => Some("gcp_trigger"), TriggerAzure => Some("azure_trigger"), diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 17d9cfd804..e2597fdfce 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -563,6 +563,20 @@ pub async fn report_critical_error( } } +/// Route a workspace-level failure to the instance critical alert channels without +/// recording an `alerts` row: job failures are workspace noise and would otherwise flood +/// the instance-wide feed superadmins triage. The channels belong to the instance operator, +/// who on cloud is not the workspace owner, hence the hard stop there. Callers own the +/// per-workspace opt-in. +pub async fn send_workspace_error_to_instance_channels(_error_message: String, _db: &DB) -> () { + if *CLOUD_HOSTED { + return; + } + + #[cfg(feature = "enterprise")] + send_critical_alert(_error_message, _db, CriticalAlertKind::CriticalError, None).await; +} + pub async fn report_recovered_critical_error( message: String, _db: DB, @@ -1044,6 +1058,92 @@ pub async fn get_custom_pg_instance_password(db: &DB) -> Result { ) } +const REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL: &str = r#" + DO $$ + DECLARE + pwd text; + BEGIN + SELECT gen_random_uuid()::text INTO pwd; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_replication_user') THEN + EXECUTE format('ALTER USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + ELSE + EXECUTE format('CREATE USER custom_instance_replication_user WITH PASSWORD %L REPLICATION', pwd); + END IF; + + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'custom_instance_user') THEN + GRANT custom_instance_user TO custom_instance_replication_user; + ALTER ROLE custom_instance_user NOREPLICATION; + END IF; + + INSERT INTO global_settings (name, value) + VALUES ('custom_instance_replication_pwd', to_jsonb(pwd::text)) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + END + $$; +"#; + +const REPLICATION_PWD_READ_SQL: &str = + "SELECT value #>> '{}' FROM global_settings WHERE name = 'custom_instance_replication_pwd'"; + +/// (Re)create `custom_instance_replication_user` with a fresh password. This role is +/// used by postgres trigger connections on custom-instance datatables; membership in +/// `custom_instance_user` lets it manage publications on the datatable tables. +/// +/// Authorization: rotates a stored database credential and performs no authorization +/// itself — callers MUST restrict this to superadmin or internal server paths. +pub async fn refresh_custom_instance_replication_user_pwd(db: &DB) -> Result<()> { + sqlx::query(REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL) + .execute(db) + .await?; + Ok(()) +} + +/// Authorization: returns a stored database credential and performs no authorization +/// itself — callers MUST restrict this to superadmin or internal server paths (mirrors +/// [`get_custom_pg_instance_password`]). +pub async fn get_custom_pg_instance_replication_password(db: &DB) -> Result { + // Fast path: already provisioned by the migration. + if let Some(pwd) = sqlx::query_scalar::<_, Option>(REPLICATION_PWD_READ_SQL) + .fetch_optional(db) + .await? + .flatten() + { + return Ok(pwd); + } + // Self-heal when the role-creating migration was swallowed. The advisory lock + re-check + // serialize concurrent workers: otherwise two callers both rotate, and the second + // rotation invalidates the password the first already returned. Rotating and reading in + // one locked transaction keeps the decision atomic. + let mut tx = db.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('custom_instance_replication_pwd'))") + .execute(&mut *tx) + .await?; + if let Some(pwd) = sqlx::query_scalar::<_, Option>(REPLICATION_PWD_READ_SQL) + .fetch_optional(&mut *tx) + .await? + .flatten() + { + tx.commit().await?; + return Ok(pwd); + } + sqlx::query(REFRESH_CUSTOM_INSTANCE_REPLICATION_USER_SQL) + .execute(&mut *tx) + .await?; + let pwd = sqlx::query_scalar::<_, Option>(REPLICATION_PWD_READ_SQL) + .fetch_optional(&mut *tx) + .await? + .flatten() + .ok_or_else(|| { + Error::BadRequest( + "Custom instance replication user password not found, did you run migrations ?" + .to_string(), + ) + })?; + tx.commit().await?; + Ok(pwd) +} + /// Convert a JSON string to a `Box` without validation. /// /// # Safety @@ -1129,9 +1229,138 @@ pub fn merge_nested_raw_values_to_array< serde_json::value::RawValue::from_string(result).unwrap() } +/// Remove every U+0000 (NUL) from a serialized JSON document so it is safe to +/// store in a `jsonb` column, which rejects the `\u0000` escape with 22P05 +/// ("unsupported Unicode escape sequence"). A `json`-typed column accepts the +/// escape but propagates the same failure to any later `->>`/`to_jsonb`/`json`→ +/// `jsonb` conversion. +/// +/// A NUL can only appear in JSON text as a backslash-u0000 escape, and a +/// backslash only ever occurs inside a string, so one backslash-parity-aware +/// pass removes every real NUL escape — covering values and keys alike — while +/// leaving a legitimate `\\u0000` (an escaped backslash followed by the literal +/// text `u0000`, common in minified JS regexes) intact. O(n) over the bytes with +/// no `serde_json::Value` tree to allocate, and the fast path (no such substring +/// at all) returns the input borrowed and untouched. The slow path is reached +/// not only by genuinely poisoned values but by any value that legitimately +/// contains `u0000` after a backslash (e.g. script source), so it must stay +/// allocation-light for potentially large documents. +pub fn strip_json_nul(serialized: &str) -> Cow<'_, str> { + // SIMD substring scan (several times faster than `str::contains`'s Two-Way) + // for the guard, since this runs on every completed job's serialized result. + if memchr::memmem::find(serialized.as_bytes(), b"\\u0000").is_none() { + return Cow::Borrowed(serialized); + } + let bytes = serialized.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + // The substring guard above is satisfied by legitimate `\\u0000` too, so only + // an odd-parity NUL escape actually drops bytes. Borrow back out when nothing + // was stripped, so `Cow::Owned` reliably means "a NUL was removed" — callers + // (e.g. apps.rs) key a warning on that. + let mut stripped = false; + while i < bytes.len() { + if bytes[i] != b'\\' { + out.push(bytes[i]); + i += 1; + continue; + } + // Consume the whole run of backslashes. An even run is N/2 escaped + // backslashes and leaves the next char unescaped; an odd run ends in an + // escaping backslash, so a following `u0000` is a real NUL escape. + let run_start = i; + while i < bytes.len() && bytes[i] == b'\\' { + i += 1; + } + let run = i - run_start; + if run % 2 == 1 && bytes[i..].starts_with(b"u0000") { + // Drop the escaping backslash + `u0000`; keep the leading literal pairs. + out.extend(std::iter::repeat(b'\\').take(run - 1)); + i += 5; + stripped = true; + } else { + out.extend(std::iter::repeat(b'\\').take(run)); + } + } + if !stripped { + return Cow::Borrowed(serialized); + } + // Only whole ASCII backslash-u0000 escapes were removed, so the bytes remain + // valid UTF-8 (and valid JSON). + Cow::Owned(String::from_utf8(out).expect("removing a NUL escape preserves valid UTF-8")) +} + #[cfg(test)] mod tests { use super::*; + + // The 6-char JSON escape for U+0000: backslash + "u0000". Written via an + // escaped backslash so no literal NUL byte ever appears in this source. + const NUL_ESC: &str = "\\u0000"; + + // Parse the (NUL-free) result so assertions read clearly. + fn parsed(s: &str) -> serde_json::Value { + serde_json::from_str(s).expect("strip_json_nul must return valid JSON") + } + + #[test] + fn strip_json_nul_clean_value_is_borrowed_byte_for_byte() { + let s = r#"{"summary":"all good","n":1}"#; + let out = strip_json_nul(s); + assert!(matches!(out, Cow::Borrowed(_))); + assert_eq!(out, s); + } + + #[test] + fn strip_json_nul_real_nul_in_value_is_stripped() { + let input = format!(r#"{{"summary":"hi{NUL_ESC}there"}}"#); + let out = strip_json_nul(&input); + assert!(!out.contains(NUL_ESC)); + assert_eq!(parsed(&out)["summary"], "hithere"); + } + + #[test] + fn strip_json_nul_legit_escaped_backslash_is_a_noop() { + // JSON "a\\u0000b" decodes to a,backslash,u,0,0,0,0,b - not a NUL - so + // the value is already clean and round-trips byte-for-byte. It hits the + // slow path (the substring is present) but strips nothing, so it must + // still return Cow::Borrowed - callers key a "stripped NUL" warning on + // the Owned variant. + let s = r#"{"summary":"a\\u0000b"}"#; + let out = strip_json_nul(s); + assert!(matches!(out, Cow::Borrowed(_))); + assert_eq!(out, s); + } + + #[test] + fn strip_json_nul_collision_real_and_literal_both_handled() { + // "a" carries a real NUL escape; "b" carries the literal text backslash-u0000. + let v = parsed(&strip_json_nul(&format!( + r#"{{"a":"x{NUL_ESC}y","b":"p\\u0000q"}}"# + ))); + assert_eq!(v["a"], "xy"); + assert_eq!(v["b"], "p\\u0000q"); + } + + #[test] + fn strip_json_nul_nested_values_and_keys_are_cleaned() { + let input = + format!(r#"{{"o":{{"k{NUL_ESC}":["a{NUL_ESC}b",{{"deep{NUL_ESC}":"v{NUL_ESC}"}}]}}}}"#); + let out = strip_json_nul(&input); + assert!(!out.contains(NUL_ESC)); + let v = parsed(&out); + assert_eq!(v["o"]["k"][0], "ab"); + assert_eq!(v["o"]["k"][1]["deep"], "v"); + } + + #[test] + fn strip_json_nul_odd_backslash_run_keeps_literal_drops_nul() { + // JSON "a\\ b" is an escaped backslash (kept) immediately followed + // by a real NUL escape (dropped) -> decodes to a,backslash,b. + let v = parsed(&strip_json_nul(&format!(r#"{{"x":"a\\{NUL_ESC}b"}}"#))); + assert_eq!(v["x"], "a\\b"); + } + #[test] fn test_build_arg_str() { let r = build_arg_str( diff --git a/backend/windmill-common/src/wac.rs b/backend/windmill-common/src/wac.rs index 7f12dfbcdb..51abc5d60f 100644 --- a/backend/windmill-common/src/wac.rs +++ b/backend/windmill-common/src/wac.rs @@ -34,6 +34,13 @@ pub struct WacCheckpoint { #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub _executing_key: Option, + /// `resume_job.id` values already consumed by earlier approval steps (the + /// row primary key, not the distinct integer `resume_id` column). Rows are + /// never deleted, so a workflow with several sequential wait_for_approval() + /// calls accumulates one per approval; excluding these lets each step read + /// its own row rather than the oldest. + #[serde(default)] + pub consumed_resume_row_ids: Vec, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -43,6 +50,38 @@ pub struct WacPendingSteps { pub job_ids: serde_json::Map, } +/// `resume_id` bound to a WAC `wait_for_approval` step key. +/// +/// Two callers must agree on it — the worker minting the inline resume/cancel +/// buttons at suspend time, and the API signing URLs the workflow asked for +/// ahead of time — so the derivation must be stable across processes and +/// releases. `DefaultHasher` is explicitly not (std makes no cross-release +/// guarantee), hence SHA-256 truncated to the `u32` the resume routes take. +/// Distinctness per key is what matters: `resume_job`'s primary key is +/// `job_id ^ resume_id`, so two steps sharing a resume_id would collide on +/// one row. +pub fn approval_resume_id(step_key: &str) -> u32 { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(step_key.as_bytes()); + u32::from_be_bytes([digest[0], digest[1], digest[2], digest[3]]) +} + +#[cfg(test)] +mod tests { + use super::approval_resume_id; + + /// Golden values: worker and API must agree on this mapping, and they can run + /// different builds during a rolling deploy. Changing it strands every resume + /// URL already in the hands of an approver, so a diff here is a deliberate + /// break, not a refactor. + #[test] + fn approval_resume_id_is_a_stable_cross_process_contract() { + assert_eq!(approval_resume_id("approval"), 0x9deb_65b8); + assert_eq!(approval_resume_id("approval_2"), 0x50d1_eeca); + assert_eq!(approval_resume_id("manager"), 0x6ee4_a469); + } +} + /// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`. pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result { let row: Option> = sqlx::query_scalar( diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ea2b5455f3..b9dd29c00a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -52,10 +52,10 @@ impl CustomTags { let tag_name = cap.get(1).unwrap().as_str().to_string(); let workspace_str = cap.get(2).unwrap().as_str(); let tag_type = SpecificTagType::from_regex_string(workspace_str); - let workspaces: Vec = workspace_str + let workspaces: Vec = workspace_str .split(tag_type.corresponding_separator()) .filter(|s| !s.is_empty()) - .map(str::to_string) + .map(WorkspaceMatcher::parse) .collect(); if workspaces.is_empty() { tracing::warn!("Ignoring tag `{}` with empty exclusion/inclusion list", e); @@ -70,11 +70,13 @@ impl CustomTags { Self { global, specific } } - pub fn to_string_vec(&self, filter_with_workspace: Option) -> Vec { - let specific = if let Some(workspace) = filter_with_workspace { + /// `filter_with_workspace` is the workspace's id chain (see [`SpecificTagData::applies_to_workspace`]); + /// `None` re-emits the authored `tag(ws1+ws2)` strings for the settings editor. + pub fn to_string_vec(&self, filter_with_workspace: Option<&[String]>) -> Vec { + let specific = if let Some(chain) = filter_with_workspace { self.specific .iter() - .filter(|(_, tag_data)| tag_data.applies_to_workspace(&workspace)) + .filter(|(_, tag_data)| tag_data.applies_to_workspace(chain)) .map(|(tag, _)| tag.clone()) .collect::>() } else { @@ -82,7 +84,12 @@ impl CustomTags { .iter() .map(|(tag, tag_data)| { let separator = tag_data.tag_type.corresponding_separator(); - let mut workspaces = tag_data.workspaces.join(&*separator.to_string()); + let mut workspaces = tag_data + .workspaces + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(&*separator.to_string()); if tag_data.tag_type == SpecificTagType::AllExcluding { // the AllExcluding tag syntax has a leading separator workspaces.insert(0, separator); @@ -95,19 +102,86 @@ impl CustomTags { all_tags.into_iter().chain(specific.into_iter()).collect() } } + +/// Marker suffixed to a workspace id inside a custom tag's scope (`mytag(prod*)`) to extend the +/// entry to that workspace's forks. `*` cannot appear in a workspace id (the `proper_id` check +/// constraint restricts them to `^\w+(-\w+)*$`), so it can never collide with a real id. +pub const FORK_SCOPE_MARKER: char = '*'; + +/// One workspace entry in a custom tag's scope. Bare (`prod`) matches that workspace only; +/// with the [`FORK_SCOPE_MARKER`] (`prod*`) it also matches its forks, transitively. +/// +/// The marker is opt-in in BOTH scope forms so that no existing tag string changes meaning: +/// `sensitive(^prod)` keeps excluding only `prod` itself, and `sensitive(^prod*)` is how you +/// exclude its forks too. +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct WorkspaceMatcher { + pub id: String, + pub include_forks: bool, +} + +impl WorkspaceMatcher { + fn parse(entry: &str) -> Self { + match entry.strip_suffix(FORK_SCOPE_MARKER) { + Some(id) => Self { id: id.to_string(), include_forks: true }, + None => Self { id: entry.to_string(), include_forks: false }, + } + } + + fn matches(&self, workspace_id: &str, fork_ancestors: &[String]) -> bool { + workspace_id == self.id + || (self.include_forks && fork_ancestors.iter().any(|a| *a == self.id)) + } +} + +impl std::fmt::Display for WorkspaceMatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id)?; + if self.include_forks { + f.write_str(FORK_SCOPE_MARKER.encode_utf8(&mut [0u8; 4]))?; + } + Ok(()) + } +} + +/// Renders the authored `prod` / `prod*` form rather than the struct fields: `CustomTags` is +/// `{:?}`-dumped into the "tag is not in the allowed CUSTOM_TAGS" error operators see. +impl std::fmt::Debug for WorkspaceMatcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(&self.to_string(), f) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct SpecificTagData { pub tag_type: SpecificTagType, - pub workspaces: Vec, + pub workspaces: Vec, } impl SpecificTagData { - pub fn applies_to_workspace(&self, workspace_id: &str) -> bool { + /// `chain` is the workspace itself followed by its fork ancestors, nearest-first, as built by + /// `workspaces::workspace_with_fork_ancestors`. Pass a single-element slice when + /// [`Self::is_fork_scoped`] is false: the ancestors cannot affect the outcome then. + pub fn applies_to_workspace(&self, chain: &[String]) -> bool { + let Some((workspace_id, fork_ancestors)) = chain.split_first() else { + return false; + }; + let matched = self + .workspaces + .iter() + .any(|w| w.matches(workspace_id, fork_ancestors)); match self.tag_type { - SpecificTagType::AllExcluding => !self.workspaces.contains(&workspace_id.to_string()), - SpecificTagType::NoneExcept => self.workspaces.contains(&workspace_id.to_string()), + SpecificTagType::AllExcluding => !matched, + SpecificTagType::NoneExcept => matched, } } + + /// Whether any entry carries the fork marker, i.e. whether resolving the workspace's fork + /// lineage can change what [`Self::applies_to_workspace`] returns. Lets hot callers skip the + /// lineage lookup for the (overwhelmingly common) fork-agnostic tag. + pub fn is_fork_scoped(&self) -> bool { + self.workspaces.iter().any(|w| w.include_forks) + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum SpecificTagType { @@ -136,6 +210,13 @@ impl SpecificTagType { pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900; pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days pub const MIN_PERIODIC_SCRIPT_INTERVAL_SECONDS: u64 = 60; +/// Default for [`CONCURRENCY_KEY_MAX_QUEUED`]; also the value the setting loader restores when +/// the setting is cleared or malformed. +pub const CONCURRENCY_KEY_MAX_QUEUED_DEFAULT: u32 = 10_000; +/// Default for [`WORKSPACE_MAX_QUEUED_JOBS`]; also the value the setting loader restores when +/// the setting is cleared or malformed. A workspace spans many keys, so this sits well above +/// the per-key cap. +pub const WORKSPACE_MAX_QUEUED_JOBS_DEFAULT: u32 = 20_000; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| { #[cfg(not(feature = "enterprise"))] @@ -263,6 +344,19 @@ lazy_static::lazy_static! { /// `should_admit_capped` is moot and "admit all" is the correct no-op. pub static ref WORKSPACE_FAIRNESS_ADMISSION_PPM: AtomicU32 = AtomicU32::new(10_000); + /// Cloud-only ceiling on the number of jobs queued behind a single concurrency key. + /// A concurrency-limited key drains at most `concurrent_limit` jobs per window, so a + /// producer pushing faster than that grows an unbounded backlog that no amount of + /// spare worker capacity can absorb. `0` disables the cap. + pub static ref CONCURRENCY_KEY_MAX_QUEUED: AtomicU32 = + AtomicU32::new(CONCURRENCY_KEY_MAX_QUEUED_DEFAULT); + + /// Cloud-only ceiling on the total number of jobs a workspace may have queued at once, + /// across every concurrency key and script. Guards against a workspace flooding the queue + /// generally (not just behind one key), including from parallel for-loops. `0` disables it. + pub static ref WORKSPACE_MAX_QUEUED_JOBS: AtomicU32 = + AtomicU32::new(WORKSPACE_MAX_QUEUED_JOBS_DEFAULT); + pub static ref SMTP_CONFIG: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref INDEXER_CONFIG: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee(TantivyIndexerSettings::default()); @@ -295,12 +389,14 @@ lazy_static::lazy_static! { // ^([\w-]+) # Group 1: tag name // \( # Literal '(' // ( # Group 2: the full workspace list - // (?:[\w-]+\+)*[\w-]+ # NoneExcept pattern: ws1+ws2 - // | # OR - // (?:\^[\w-]+)+ # AllExcluding pattern: ^ws1^ws2 + // (?:[\w-]+\*?\+)*[\w-]+\*? # NoneExcept pattern: ws1+ws2* + // | # OR + // (?:\^[\w-]+\*?)+ # AllExcluding pattern: ^ws1^ws2* // ) // \)$ # Closing ')' - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\+)*[\w-]+|(?:\^[\w-]+)+)\)$").unwrap(); + // + // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() @@ -2332,6 +2428,19 @@ mod tests { use super::*; use std::collections::HashMap; + fn matcher(id: &str) -> WorkspaceMatcher { + WorkspaceMatcher { id: id.to_string(), include_forks: false } + } + + fn fork_matcher(id: &str) -> WorkspaceMatcher { + WorkspaceMatcher { id: id.to_string(), include_forks: true } + } + + /// A workspace id chain: the workspace itself, then its fork ancestors nearest-first. + fn chain(ids: &[&str]) -> Vec { + ids.iter().map(|s| s.to_string()).collect() + } + #[test] fn test_bash_sandbox_image_annotation() { // `# sandbox ` selects the container runtime and returns the image. @@ -2430,14 +2539,14 @@ mod tests { "feat".to_string(), SpecificTagData { tag_type: SpecificTagType::NoneExcept, - workspaces: vec!["ws1".to_string(), "ws2".to_string()], + workspaces: vec![matcher("ws1"), matcher("ws2")], }, ); expected.insert( "hotfix".to_string(), SpecificTagData { tag_type: SpecificTagType::AllExcluding, - workspaces: vec!["ws3".to_string(), "ws4".to_string()], + workspaces: vec![matcher("ws3"), matcher("ws4")], }, ); @@ -2480,7 +2589,7 @@ mod tests { let data = tags.specific.get("urgent").unwrap(); assert_eq!(data.tag_type, SpecificTagType::NoneExcept); - assert_eq!(data.workspaces, vec!["ws1", "ws2"]); + assert_eq!(data.workspaces, vec![matcher("ws1"), matcher("ws2")]); } #[test] @@ -2493,7 +2602,7 @@ mod tests { let data = tags.specific.get("legacy").unwrap(); assert_eq!(data.tag_type, SpecificTagType::AllExcluding); - assert_eq!(data.workspaces, vec!["ws1", "ws2"]); + assert_eq!(data.workspaces, vec![matcher("ws1"), matcher("ws2")]); } #[test] @@ -2510,10 +2619,10 @@ mod tests { let input = vec!["urgent(ws1+ws2)".to_string()]; let tags = CustomTags::from(input); - let output = tags.to_string_vec(Some("ws1".to_string())); + let output = tags.to_string_vec(Some(&chain(&["ws1"]))); assert_eq!(output, vec!["urgent"]); - let output_none = tags.to_string_vec(Some("ws3".to_string())); + let output_none = tags.to_string_vec(Some(&chain(&["ws3"]))); assert!(output_none.is_empty()); } @@ -2522,10 +2631,10 @@ mod tests { let input = vec!["legacy(^ws1^ws2)".to_string()]; let tags = CustomTags::from(input); - let output = tags.to_string_vec(Some("ws3".to_string())); + let output = tags.to_string_vec(Some(&chain(&["ws3"]))); assert_eq!(output, vec!["legacy"]); - let output_excluded = tags.to_string_vec(Some("ws1".to_string())); + let output_excluded = tags.to_string_vec(Some(&chain(&["ws1"]))); assert!(output_excluded.is_empty()); } @@ -2543,6 +2652,68 @@ mod tests { assert_eq!(result, vec!["foo", "legacy(^ws1^ws2)", "urgent(ws1+ws2)"]); } + #[test] + fn test_fork_marker_parses_and_round_trips() { + let tags = CustomTags::from(vec![ + "urgent(prod*+ws2)".to_string(), + "legacy(^prod*)".to_string(), + ]); + + let urgent = tags.specific.get("urgent").unwrap(); + assert_eq!( + urgent.workspaces, + vec![fork_matcher("prod"), matcher("ws2")] + ); + assert!(urgent.is_fork_scoped()); + + let legacy = tags.specific.get("legacy").unwrap(); + assert_eq!(legacy.workspaces, vec![fork_matcher("prod")]); + + // The settings editor re-emits what it parsed; dropping `*` here would silently widen + // an excluding tag / narrow an including one on every save. + let mut result = tags.to_string_vec(None); + result.sort(); + assert_eq!(result, vec!["legacy(^prod*)", "urgent(prod*+ws2)"]); + } + + #[test] + fn test_fork_marker_extends_none_except_to_forks_only_when_present() { + let fork = chain(&["wm-fork-x", "prod"]); + let nested = chain(&["wm-fork-y", "wm-fork-x", "prod"]); + + let marked = CustomTags::from(vec!["urgent(prod*)".to_string()]); + let marked = marked.specific.get("urgent").unwrap(); + assert!(marked.applies_to_workspace(&chain(&["prod"]))); + assert!(marked.applies_to_workspace(&fork)); + assert!(marked.applies_to_workspace(&nested)); + assert!(!marked.applies_to_workspace(&chain(&["wm-fork-z", "other"]))); + + // Without the marker a fork must NOT inherit the parent's tag. + let unmarked = CustomTags::from(vec!["urgent(prod)".to_string()]); + let unmarked = unmarked.specific.get("urgent").unwrap(); + assert!(unmarked.applies_to_workspace(&chain(&["prod"]))); + assert!(!unmarked.applies_to_workspace(&fork)); + // Gates the ancestor lookup, so a wrong answer here silently disables the marker. + assert!(!unmarked.is_fork_scoped()); + } + + #[test] + fn test_fork_marker_extends_all_excluding_to_forks_only_when_present() { + let fork = chain(&["wm-fork-x", "prod"]); + + // Pre-existing exclusions keep their exact meaning: only `prod` itself is excluded. + let unmarked = CustomTags::from(vec!["legacy(^prod)".to_string()]); + let unmarked = unmarked.specific.get("legacy").unwrap(); + assert!(!unmarked.applies_to_workspace(&chain(&["prod"]))); + assert!(unmarked.applies_to_workspace(&fork)); + + let marked = CustomTags::from(vec!["legacy(^prod*)".to_string()]); + let marked = marked.specific.get("legacy").unwrap(); + assert!(!marked.applies_to_workspace(&chain(&["prod"]))); + assert!(!marked.applies_to_workspace(&fork)); + assert!(marked.applies_to_workspace(&chain(&["other"]))); + } + #[test] fn test_dedicated_worker_tag_short() { let tag = dedicated_worker_tag("demo", "u/alice/script"); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index a8c6e26f0e..135b8b9558 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -9,7 +9,7 @@ use crate::{ error::{self, to_anyhow, Error, Result}, get_database_url, secret_backend::{get_secret_value, is_external_stored_value}, - utils::get_custom_pg_instance_password, + utils::{get_custom_pg_instance_password, get_custom_pg_instance_replication_password}, variables::{build_crypt, decrypt}, PgDatabase, DB, }; @@ -167,7 +167,15 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28719/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28809/sync-script-to-git-repo-windmill"; + +/// Hub script that applies a repository's state back into a workspace +/// (the repo → Windmill / "pull" direction). Same script the UI runs from +/// `PullWorkspaceModal` with `pull: true`. The hub resolves by numeric id and +/// ignores the slug, so the slug is kept free of characters that would be +/// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened +/// reverse proxies reject as double-encoding when the client re-encodes it). +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28808/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -190,6 +198,39 @@ pub fn validate_dev_workspace_id(id: &str) -> error::Result<()> { validate_workspace_branch_id(id, false) } +/// Split a fork git branch `wm-fork//` into `(base_branch, suffix)`. +/// +/// Inverse of the CLI/hub-script `forkBranchName`. The suffix is a workspace id +/// fragment and can't contain `/` (enforced by [`validate_fork_workspace_id`] / +/// [`validate_dev_workspace_id`] at every fork/dev creation and attach path), while +/// the base branch may (`release/v2`), so the split is on the last separator. +/// Returns `None` for anything else. +pub fn parse_fork_branch(branch: &str) -> Option<(&str, &str)> { + let rest = branch.strip_prefix("wm-fork/")?; + let idx = rest.rfind('/')?; + let (base, suffix) = (&rest[..idx], &rest[idx + 1..]); + if base.is_empty() || suffix.is_empty() { + return None; + } + Some((base, suffix)) +} + +/// Workspace ids that could own a fork branch with this suffix: a generated fork +/// (`wm-fork-`, whose branch strips the id prefix) or a dev workspace +/// (prefix-less id used verbatim). Ordered generated-fork first so an ambiguous +/// suffix resolves deterministically. +pub fn fork_branch_workspace_id_candidates(suffix: &str) -> [String; 2] { + [format!("{WM_FORK_PREFIX}{suffix}"), suffix.to_string()] +} + +/// Git branch a dev workspace syncs with: its environment label verbatim, as a +/// first-class top-level branch (`dev`, `staging` — the classic env-branch +/// layout), defaulting to `dev` when the label is unset. Throwaway forks use +/// the namespaced `wm-fork//` form instead. +pub fn dev_workspace_branch(label: Option<&str>) -> String { + label.filter(|l| !l.is_empty()).unwrap_or("dev").to_string() +} + /// The `workspace.name` column is `character varying(50)`, so a name longer than 50 characters /// triggers a raw `value too long for type character varying(50)` SQL error on insert. Validate /// up front to return a clear message instead. @@ -244,12 +285,13 @@ fn validate_workspace_branch_id(id: &str, require_fork_prefix: bool) -> error::R if id.contains("@{") { return reject("cannot contain '@{'"); } - if id.contains("//") { - return reject("cannot contain '//'"); - } for ch in id.chars() { match ch { - ':' | '~' | '^' | '?' | '*' | '[' | '\\' | ' ' => { + // '/' is git-legal in a branch but banned here: the id becomes the last + // component of `wm-fork//` and `parse_fork_branch` splits + // that branch on the last '/' (the base branch itself may contain '/'), + // so a slash in the id would make the fork unroutable for auto-sync. + ':' | '~' | '^' | '?' | '*' | '[' | '\\' | ' ' | '/' => { return reject(&format!("contains forbidden character '{}'", ch)); } c if c.is_ascii_control() || c == '\u{7f}' => { @@ -258,18 +300,16 @@ fn validate_workspace_branch_id(id: &str, require_fork_prefix: bool) -> error::R _ => {} } } - // Each slash-separated component cannot start with '.' or end with '.lock'. - for component in id.split('/') { - if component.starts_with('.') { - return reject("a path component cannot start with '.'"); - } - if component.ends_with(".lock") { - return reject("a path component cannot end with '.lock'"); - } + if id.starts_with('.') { + return reject("cannot start with '.'"); } Ok(()) } +fn is_false(b: &bool) -> bool { + !*b +} + #[derive(Serialize, Deserialize, Debug)] pub struct GitRepositorySettings { #[serde(skip_serializing_if = "Option::is_none")] @@ -283,6 +323,27 @@ pub struct GitRepositorySettings { pub group_by_folder: Option, #[serde(skip_serializing_if = "Option::is_none")] pub settings: Option, + /// Configuration for automatically pulling changes from the git repository + /// back into the workspace (repo → Windmill direction). Absent means the + /// reverse direction is not automated (the historical behaviour). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auto_pull: Option, + /// Open a PR when a deploy pushes a `wm_deploy/**` branch of this promotion + /// repo (app-backed only; runs from the deploy callback so it works without + /// inbound webhooks). Off by default so upgrades don't change behavior. + #[serde(default, skip_serializing_if = "is_false")] + pub promotion_open_prs: bool, + /// Parent-level: open a PR when a fork of this workspace deploys to its + /// `wm-fork/**` branch (app-backed only; the fork's deploy callback reads + /// this from the parent). Off by default. + #[serde(default, skip_serializing_if = "is_false")] + pub fork_open_prs: bool, + /// Server-owned: the last failure opening a PR for a deploy branch of this + /// repo (e.g. the GitHub App installation hasn't approved the pull-request + /// permission). Written by the deploy completion hook, cleared on the next + /// successful PR; never accepted from clients. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub open_pr_error: Option, } impl GitRepositorySettings { @@ -314,6 +375,128 @@ impl GitRepositorySettings { } } +/// How auto-pull triggers are delivered for a repository. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum AutoPullMode { + /// Try to create a repo webhook; fall back to polling if the instance is not + /// reachable from GitHub or the app lacks the webhook permission. + #[default] + Auto, + /// Webhook delivery only (no polling fallback). + Webhook, + /// Polling only (`git ls-remote` on an interval). + Polling, +} + +/// Outcome of the most recent auto-pull attempt, surfaced in the UI. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AutoPullStatus { + /// Commit sha the workspace was last synced to. + #[serde(skip_serializing_if = "Option::is_none")] + pub synced_sha: Option, + /// Unix timestamp (seconds) of the attempt. + pub at: i64, + /// Job id of the pull run, if one was enqueued. + #[serde(skip_serializing_if = "Option::is_none")] + pub job_id: Option, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +/// Per-repository configuration for automatic repo → Windmill pull sync. +/// +/// Stored inside `GitRepositorySettings` (workspace_settings.git_sync JSONB). +/// Webhook fields are populated in phase 2; phase 1 exercises the polling path +/// only, but the full shape is defined up front to avoid a second schema change. +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct AutoPullSettings { + /// Default so a server-written status-only blob (fork workspaces, which never + /// enable auto-pull themselves) parses even without the field. + #[serde(default)] + pub enabled: bool, + #[serde(default)] + pub mode: AutoPullMode, + /// Polling interval in seconds. Defaults to `DEFAULT_AUTO_PULL_POLL_INTERVAL_S` + /// when polling without an active webhook, relaxed once a webhook is live. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub poll_interval_s: Option, + /// Parent-level: also pull each live fork of this workspace from its own + /// `wm-fork//` branch when that branch moves (webhook or poll), + /// the managed equivalent of the `push-on-merge-to-forks` GitHub Action. + /// Configured once on the parent; forks never enable auto-pull themselves. + #[serde(default, skip_serializing_if = "is_false")] + pub sync_forks: bool, + /// GitHub repository webhook id (managed-app, phase 2). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub webhook_id: Option, + /// HMAC secret for the repo webhook, encrypted at rest (managed-app, phase 2). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub webhook_secret: Option, + /// Why the repo has no active webhook while one was requested (auto/webhook + /// mode): instance base URL unset, app missing the webhook permission, etc. + /// Surfaced in the UI as a "falling back to polling" warning; `None` when the + /// webhook is live or the repo is polling-only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub webhook_error: Option, + /// Last synced commit sha per tracked git ref (e.g. `refs/heads/main`). + #[serde(default, skip_serializing_if = "std::collections::HashMap::is_empty")] + pub last_synced_sha: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_pull_status: Option, +} + +// Manual Debug so the HMAC `webhook_secret` (even encrypted) never lands in logs. +impl std::fmt::Debug for AutoPullSettings { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AutoPullSettings") + .field("enabled", &self.enabled) + .field("mode", &self.mode) + .field("poll_interval_s", &self.poll_interval_s) + .field("sync_forks", &self.sync_forks) + .field("webhook_id", &self.webhook_id) + .field( + "webhook_secret", + &self.webhook_secret.as_ref().map(|_| ""), + ) + .field("webhook_error", &self.webhook_error) + .field("last_synced_sha", &self.last_synced_sha) + .field("last_pull_status", &self.last_pull_status) + .finish() + } +} + +/// Default polling interval when a webhook is not active. +pub const DEFAULT_AUTO_PULL_POLL_INTERVAL_S: u32 = 60; + +/// Relaxed polling interval used as a safety net while a webhook is active. +pub const WEBHOOK_AUTO_PULL_POLL_INTERVAL_S: u32 = 600; + +impl AutoPullSettings { + /// Effective polling interval in seconds, honouring the explicit override and + /// relaxing to `WEBHOOK_AUTO_PULL_POLL_INTERVAL_S` when a webhook is live. + pub fn effective_poll_interval_s(&self) -> u32 { + self.poll_interval_s.unwrap_or_else(|| { + if self.webhook_id.is_some() { + WEBHOOK_AUTO_PULL_POLL_INTERVAL_S + } else { + DEFAULT_AUTO_PULL_POLL_INTERVAL_S + } + }) + } + + /// Whether a freshly observed `(git_ref, head_sha)` warrants enqueuing a pull. + /// + /// A trigger (poll or webhook) is only a hint: we pull when auto-pull is + /// enabled and the observed head differs from the last sha we synced for + /// that ref. Re-observing the same head (e.g. a redundant poll, or the + /// commit our own deploy callback just pushed back) is a no-op. + pub fn should_pull(&self, git_ref: &str, head_sha: &str) -> bool { + self.enabled && self.last_synced_sha.get(git_ref).map(String::as_str) != Some(head_sha) + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct GitSyncSettings { pub include_path: Vec, @@ -472,6 +655,35 @@ pub async fn fork_subtree_height(db: &crate::DB, w_id: &str) -> Result { Ok(height) } +/// Parent id of `w_id` when `email` is the creator of that fork, `None` otherwise (including for a +/// root workspace, which has no creator in this sense). +/// +/// The creator is recorded as `workspace.owner`, but the `usr` row they get in the fork is copied +/// from the parent — so a forker who is not an admin of the parent is not an admin of the fork they +/// just created either, and cannot bring anyone in to work on it. Being the creator therefore grants +/// a narrow membership right over the fork; the callers own the exact bounds of that grant (see +/// `add_user` in `windmill-api-workspaces`). +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). Takes any executor so that a +/// caller can run it inside the transaction whose writes the grant authorizes. +pub async fn fork_owned_by<'e, E: sqlx::Executor<'e, Database = sqlx::Postgres>>( + db: E, + w_id: &str, + email: &str, +) -> Result> { + let parent = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace + WHERE id = $1 AND owner = $2 AND parent_workspace_id IS NOT NULL AND NOT deleted", + w_id, + email + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("checking fork ownership of {w_id}: {e:#}")))?; + Ok(parent.flatten()) +} + /// Ids of every fork/dev workspace anywhere under `w_id` (excludes `w_id` itself), including live /// descendants beneath a soft-deleted intermediate. Used to invalidate per-workspace caches for a /// whole subtree after its ancestor is reparented. @@ -832,6 +1044,32 @@ pub async fn get_datatable_resource_from_db_unchecked( db: &DB, w_id: &str, name: &str, +) -> Result { + get_datatable_resource_inner(db, w_id, name, false).await +} + +/// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger +/// connections: custom-instance datatables resolve to +/// `custom_instance_replication_user` rather than `custom_instance_user`. BYO-postgres +/// datatables resolve to the user's own resource unchanged; configuring it for +/// replication there is the user's responsibility. +/// +/// Authorization: like its `_unchecked` sibling, returns resolved connection +/// credentials and performs no authorization — callers MUST have already authorized +/// access to the datatable (e.g. the trigger's own create-time check). +pub async fn get_datatable_replication_resource_from_db_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result { + get_datatable_resource_inner(db, w_id, name, true).await +} + +async fn get_datatable_resource_inner( + db: &DB, + w_id: &str, + name: &str, + replication: bool, ) -> Result { let datatables = sqlx::query_scalar!( r#" @@ -856,8 +1094,13 @@ pub async fn get_datatable_resource_from_db_unchecked( { let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; pg_creds.dbname = datatable.database.resource_path.clone(); - pg_creds.user = Some("custom_instance_user".to_string()); - pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); + if replication { + pg_creds.user = Some("custom_instance_replication_user".to_string()); + pg_creds.password = Some(get_custom_pg_instance_replication_password(&db).await?); + } else { + pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); + } serde_json::to_value(&pg_creds) .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))? } else { @@ -1211,6 +1454,108 @@ pub async fn fork_ancestor_chain(db: &crate::DB, w_id: &str) -> Result Result> { + let mut chain = Vec::with_capacity(4); + chain.push(w_id.to_string()); + chain.extend(fork_ancestor_chain(db, w_id).await?); + Ok(chain) +} + +/// Resolve which live descendant workspace (and its inherited repo entry) a git +/// branch pushed to `parent_repo_path` — a git-sync repo on `parent_w_id` +/// tracking `expected_base` — deploys to via parent-managed fork sync, or `None` +/// if it routes to no fork. Shared by the auto-pull reconciler and deploy-mode +/// detection so both agree on fork/dev routing. +/// +/// Reads lineage/settings for arbitrary ids with no authz check (like +/// [`fork_ancestor_chain`]); the caller must already be authorized for the +/// workspace whose deploy path it is resolving. +pub async fn resolve_fork_branch_target( + db: &DB, + parent_w_id: &str, + parent_repo_path: &str, + branch: &str, + expected_base: &str, +) -> Result> { + // Only a live descendant of parent_w_id may receive the pull — a crafted + // branch name must not route into an unrelated workspace. Descendants (not + // just direct children) because a fork of a dev workspace also syncs through + // the root's webhook/poller: only the root can hold auto-pull config. + let fork_id: Option = if let Some((base, suffix)) = parse_fork_branch(branch) { + // Throwaway-fork form derives from the tracked branch + // (`wm-fork//`); a different base is not this repo's. + if base != expected_base { + return Ok(None); + } + let [generated_id, dev_id] = fork_branch_workspace_id_candidates(suffix); + sqlx::query_scalar!( + r#"WITH RECURSIVE descendants AS ( + SELECT id, 0 AS depth FROM workspace + WHERE parent_workspace_id = $1 AND NOT deleted + UNION ALL + SELECT w.id, d.depth + 1 FROM workspace w + JOIN descendants d ON w.parent_workspace_id = d.id + WHERE NOT w.deleted AND d.depth < 10 + ) + SELECT id as "id!" FROM descendants WHERE (id = $2 OR id = $3) + ORDER BY (id = $2) DESC LIMIT 1"#, + parent_w_id, + generated_id, + dev_id, + ) + .fetch_optional(db) + .await? + } else if branch != expected_base { + // Environment-label branch (`dev`/`staging`) of a dev-workspace child. + // Dev workspaces only exist directly under a root, so no recursion here. + // The tracked-branch guard keeps a label that collides with the tracked + // branch from double-routing (the parent's own pull already covers it). + sqlx::query_scalar!( + "SELECT id FROM workspace \ + WHERE parent_workspace_id = $1 AND NOT deleted AND is_dev_workspace \ + AND COALESCE(dev_workspace_label, 'dev') = $2", + parent_w_id, + branch, + ) + .fetch_optional(db) + .await? + } else { + return Ok(None); + }; + let Some(fork_id) = fork_id else { + return Ok(None); + }; + + // The fork inherited the repo entry at fork time; use its own copy. + let fork_repo = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + &fork_id + ) + .fetch_optional(db) + .await? + .flatten() + .and_then(|v| serde_json::from_value::(v).ok()) + .and_then(|s| { + s.repositories + .into_iter() + .find(|r| r.git_repo_resource_path == parent_repo_path) + }); + if fork_repo.is_none() { + tracing::warn!( + "git fork sync: fork {fork_id} has no git-sync repo {parent_repo_path}, not routing {branch}" + ); + } + Ok(fork_repo.map(|r| (fork_id, r))) +} + pub async fn get_ducklake_from_db_unchecked( name: &str, w_id: &str, @@ -1783,6 +2128,33 @@ async fn transform_json_unchecked( mod tests { use super::*; + #[test] + fn test_parse_fork_branch() { + // Generated fork (`wm-fork-abc`) and dev workspace (`staging`) forms. + assert_eq!(parse_fork_branch("wm-fork/main/abc"), Some(("main", "abc"))); + assert_eq!( + parse_fork_branch("wm-fork/main/staging"), + Some(("main", "staging")) + ); + // Base branch may itself contain slashes; the suffix never does. + assert_eq!( + parse_fork_branch("wm-fork/release/v2/abc"), + Some(("release/v2", "abc")) + ); + assert_eq!(parse_fork_branch("main"), None); + assert_eq!(parse_fork_branch("wm-fork/main"), None); + assert_eq!(parse_fork_branch("wm-fork/main/"), None); + assert_eq!(parse_fork_branch("wm-fork//abc"), None); + assert_eq!(parse_fork_branch("wm_deploy/main/abc"), None); + } + + #[test] + fn test_fork_branch_workspace_id_candidates_prefers_generated_fork() { + let [first, second] = fork_branch_workspace_id_candidates("abc"); + assert_eq!(first, "wm-fork-abc"); + assert_eq!(second, "abc"); + } + #[test] fn test_validate_fork_workspace_id_accepts_valid() { validate_fork_workspace_id("wm-fork-test-allow").unwrap(); @@ -1808,6 +2180,9 @@ mod tests { "wm-fork-test[allow", "wm-fork-test\\allow", "wm-fork-test\nallow", + // '/' is git-legal but banned: it would break the parse_fork_branch + // last-'/' split that routes auto-pull into the fork workspace. + "wm-fork-test/allow", ] { assert!( validate_fork_workspace_id(bad).is_err(), @@ -1858,6 +2233,59 @@ mod tests { assert!(validate_fork_workspace_id(&long_id).is_err()); } + fn auto_pull(enabled: bool, synced: &[(&str, &str)]) -> AutoPullSettings { + AutoPullSettings { + enabled, + mode: AutoPullMode::Auto, + poll_interval_s: None, + sync_forks: false, + webhook_id: None, + webhook_secret: None, + webhook_error: None, + last_synced_sha: synced + .iter() + .map(|(r, s)| (r.to_string(), s.to_string())) + .collect(), + last_pull_status: None, + } + } + + #[test] + fn test_should_pull_on_new_or_changed_sha() { + let s = auto_pull(true, &[("refs/heads/main", "aaa")]); + // unchanged head → no pull + assert!(!s.should_pull("refs/heads/main", "aaa")); + // moved head → pull + assert!(s.should_pull("refs/heads/main", "bbb")); + // never-seen ref → pull + assert!(s.should_pull("refs/heads/dev", "ccc")); + } + + #[test] + fn test_should_pull_respects_enabled_flag() { + let s = auto_pull(false, &[]); + assert!(!s.should_pull("refs/heads/main", "bbb")); + } + + #[test] + fn test_effective_poll_interval() { + let mut s = auto_pull(true, &[]); + assert_eq!( + s.effective_poll_interval_s(), + DEFAULT_AUTO_PULL_POLL_INTERVAL_S + ); + // explicit override wins + s.poll_interval_s = Some(15); + assert_eq!(s.effective_poll_interval_s(), 15); + // with a live webhook and no override, relax to the webhook interval + s.poll_interval_s = None; + s.webhook_id = Some(42); + assert_eq!( + s.effective_poll_interval_s(), + WEBHOOK_AUTO_PULL_POLL_INTERVAL_S + ); + } + #[test] fn test_fork_ducklake_metadata_schema_shape() { let s = fork_ducklake_metadata_schema("wm-fork-my-feature-42", "main"); @@ -1931,10 +2359,12 @@ mod tests { #[test] fn test_fork_data_path_prefix_isolation() { - // Fork/dev ids are git-branch-safe and may contain `/`: `wm-fork-a/b` is a valid id. - // Its data prefix must NOT nest inside `wm-fork-a`'s, or deleting `wm-fork-a` would - // sweep the sibling's files via the object-store prefix listing. - assert!(validate_fork_workspace_id("wm-fork-a/b").is_ok()); + // New fork/dev ids can't contain `/` (it would break the parse_fork_branch + // last-'/' split), but ids created before that ban may still exist, so the + // data-path mangling must keep handling them: `wm-fork-a/b`'s prefix must + // NOT nest inside `wm-fork-a`'s, or deleting `wm-fork-a` would sweep the + // sibling's files via the object-store prefix listing. + assert!(validate_fork_workspace_id("wm-fork-a/b").is_err()); let a = fork_data_path("lake", "wm-fork-a"); let ab = fork_data_path("lake", "wm-fork-a/b"); assert!( diff --git a/backend/windmill-common/tests/low_disk_alerts.rs b/backend/windmill-common/tests/low_disk_alerts.rs new file mode 100644 index 0000000000..d6cc1e5009 --- /dev/null +++ b/backend/windmill-common/tests/low_disk_alerts.rs @@ -0,0 +1,57 @@ +//! Regression test for the server-mode low-disk alert dedup tag. +//! +//! ## Requirements +//! +//! - PostgreSQL database running locally +//! - Enterprise features enabled +//! +//! ## Running the tests +//! +//! ```bash +//! cargo test -p windmill-common --test low_disk_alerts --features private,enterprise -- --ignored --nocapture +//! ``` + +#[cfg(all(feature = "private", feature = "enterprise"))] +mod tests { + use sqlx::{Pool, Postgres}; + use windmill_common::ee::low_disk_alerts; + use windmill_common::utils::HOSTNAME; + + /// The server tag must carry the hostname: `simple_alert_helper` keys one alert row per + /// tag, so a host-less tag lets a replica seeing low disk and a replica seeing free disk + /// raise and recover the same row every monitor pass. + /// + /// The hostname is forced to a pod-length name so the tag runs past 50 chars, which + /// `check_type` must stay wide enough to hold: `create_alert` only logs the insert + /// error while the notification still fires, so a tag that does not fit re-alerts every + /// pass and never recovers. Asserting the row persists pins the width and the shape. + #[ignore = "requires database setup - run with --ignored flag"] + #[sqlx::test(migrations = "../migrations")] + async fn server_low_disk_tag_is_per_host(db: Pool) { + // Both statics are lazy and read on first access inside the call below. + std::env::set_var("FORCE_HOSTNAME", "windmill-server-7d9f8b6c4d-x2k9p"); + // Force every mount to read as low so the server branch raises. + std::env::set_var("MIN_FREE_DISK_SPACE_MB", "999999999999"); + + low_disk_alerts(&db, true, false, vec![]).await; + + let tags: Vec = sqlx::query_scalar( + "SELECT check_type FROM healthchecks WHERE check_type LIKE 'low-disk-v2-server@%'", + ) + .fetch_all(&db) + .await + .unwrap(); + + assert!( + !tags.is_empty(), + "expected at least one server low-disk alert; an alert whose tag does not fit \ + check_type is dropped here while its notification still fires" + ); + for tag in &tags { + assert!( + tag.ends_with(&format!("@{}", *HOSTNAME)), + "server tag {tag} is not per-host; replicas would share one alert row" + ); + } + } +} diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 6c1b9bf89b..cf000244e5 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -14,7 +14,9 @@ pub mod git_sync_oss; #[cfg(feature = "private")] pub use git_sync_ee::{ - handle_deployment_metadata, handle_deployment_metadata_batch, handle_fork_branch_creation, + enqueue_git_pull_dry_run, enqueue_git_pull_job, handle_deployment_metadata, + handle_deployment_metadata_batch, handle_fork_branch_creation, persist_auto_pull_state, + reconcile_and_enqueue_pull, reconcile_fork_branch_pull, record_auto_pull_failure, }; #[cfg(not(feature = "private"))] @@ -91,6 +93,10 @@ pub enum DeployedObject { path: String, parent_path: Option, }, + AmqpTrigger { + path: String, + parent_path: Option, + }, SqsTrigger { path: String, parent_path: Option, @@ -142,6 +148,7 @@ impl DeployedObject { DeployedObject::NatsTrigger { path, .. } => path.to_owned(), DeployedObject::PostgresTrigger { path, .. } => path.to_owned(), DeployedObject::MqttTrigger { path, .. } => path.to_owned(), + DeployedObject::AmqpTrigger { path, .. } => path.to_owned(), DeployedObject::SqsTrigger { path, .. } => path.to_owned(), DeployedObject::GcpTrigger { path, .. } => path.to_owned(), DeployedObject::AzureTrigger { path, .. } => path.to_owned(), @@ -184,6 +191,7 @@ impl DeployedObject { DeployedObject::NatsTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::PostgresTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::MqttTrigger { parent_path, .. } => parent_path.to_owned(), + DeployedObject::AmqpTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::SqsTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::GcpTrigger { parent_path, .. } => parent_path.to_owned(), DeployedObject::AzureTrigger { parent_path, .. } => parent_path.to_owned(), @@ -214,6 +222,7 @@ impl DeployedObject { DeployedObject::NatsTrigger { .. } => "nats_trigger", DeployedObject::PostgresTrigger { .. } => "postgres_trigger", DeployedObject::MqttTrigger { .. } => "mqtt_trigger", + DeployedObject::AmqpTrigger { .. } => "amqp_trigger", DeployedObject::SqsTrigger { .. } => "sqs_trigger", DeployedObject::GcpTrigger { .. } => "gcp_trigger", DeployedObject::AzureTrigger { .. } => "azure_trigger", @@ -444,6 +453,10 @@ mod tests { DeployedObject::MqttTrigger { path: "t".to_string(), parent_path: None }.get_kind(), "mqtt_trigger" ); + assert_eq!( + DeployedObject::AmqpTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + "amqp_trigger" + ); assert_eq!( DeployedObject::SqsTrigger { path: "t".to_string(), parent_path: None }.get_kind(), "sqs_trigger" diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index 34dc141209..f61a1c3f85 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -43,7 +43,7 @@ impl McpClient { // The resource URL is author-controlled and we send a (potentially // secret) bearer token to it, so it must be validated against SSRF // before we connect (e.g. cloud metadata endpoints, internal services). - windmill_common::ssrf::validate_mcp_server_url(&resource.url) + let validated = windmill_common::ssrf::validate_mcp_server_url(&resource.url) .await .map_err(|e| { anyhow::anyhow!( @@ -76,14 +76,24 @@ impl McpClient { } } - let reqwest_client = reqwest::Client::builder() + let mut client_builder = reqwest::Client::builder() .default_headers(headers) // Don't follow redirects: the SSRF check above only validates the // initial (author-controlled) URL, so following a redirect could // still reach a private/internal address with the bearer token // attached. The MCP streamable-HTTP endpoint is a direct endpoint // and does not legitimately rely on redirects. - .redirect(reqwest::redirect::Policy::none()) + .redirect(reqwest::redirect::Policy::none()); + // Pin DNS to the address validated above so the connect cannot rebind to + // an internal IP between the check and the request. `apply_dns_pinning` + // lives on windmill-common's reqwest, but this crate resolves a + // different reqwest version (via rmcp), so pin directly with the + // std-typed host/addrs the validation surfaced. Empty addrs (IP literal + // or ALLOW_PRIVATE_MCP_SERVER_URLS) leave resolution untouched. + if !validated.addrs.is_empty() { + client_builder = client_builder.resolve_to_addrs(&validated.host, &validated.addrs); + } + let reqwest_client = client_builder .build() .context("Failed to build HTTP client")?; diff --git a/backend/windmill-mcp/src/client_registration.rs b/backend/windmill-mcp/src/client_registration.rs index b0b07cada1..637b76df80 100644 --- a/backend/windmill-mcp/src/client_registration.rs +++ b/backend/windmill-mcp/src/client_registration.rs @@ -17,7 +17,7 @@ use windmill_common::db::DB; use windmill_common::error; use windmill_common::variables::{build_crypt, decrypt, encrypt}; -use crate::oauth::{no_redirect_http_client, AuthorizationManager}; +use crate::oauth::{no_redirect_http_client_pinned, AuthorizationManager}; /// MCP client credentials returned by [`get_or_refresh_mcp_client`]. pub struct McpClientCredentials { @@ -77,13 +77,15 @@ async fn register_client( redirect_uri: &str, client_name: &str, ) -> Result { - windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + let validated = windmill_common::ssrf::validate_mcp_server_url_for_bad_request( registration_endpoint, "MCP server registration endpoint URL", ) .await?; - let client = no_redirect_http_client() + // Pin to the validated address: DCR posts to an author-controlled endpoint, + // so the connect must not rebind to an internal IP after the check. + let client = no_redirect_http_client_pinned(&validated) .map_err(|e| error::Error::BadRequest(format!("Failed to build DCR client: {e}")))?; let request = DcrRequest { client_name: client_name.to_string(), @@ -128,7 +130,7 @@ pub async fn get_or_refresh_mcp_client( let base_url = (**windmill_common::BASE_URL.load()).clone(); let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url); - windmill_common::ssrf::validate_mcp_server_url_for_bad_request( + let validated_server = windmill_common::ssrf::validate_mcp_server_url_for_bad_request( mcp_server_url, "MCP server URL", ) @@ -166,7 +168,13 @@ pub async fn get_or_refresh_mcp_client( let mut manager = AuthorizationManager::new(mcp_server_url) .await .map_err(|e| error::Error::BadRequest(format!("Failed to create auth manager: {e}")))?; - let discovery_client = no_redirect_http_client().map_err(|e| { + // Discovery hits the well-known endpoint on the MCP server host validated + // above; pin to that address so it cannot rebind between check and connect. + // Limitation: rmcp's discover_metadata may additionally follow server-supplied + // metadata URLs (resource_metadata / authorization_servers) on other hosts, + // which this per-host pin does not cover — a pre-existing gap in rmcp discovery + // that a validating resolver would need to close, out of scope for this pin. + let discovery_client = no_redirect_http_client_pinned(&validated_server).map_err(|e| { error::Error::BadRequest(format!("Failed to build MCP OAuth discovery client: {e}")) })?; manager diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index d9cae883de..d5eb0b9597 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -57,6 +57,27 @@ pub mod oauth { .build() } + /// Like [`no_redirect_http_client`], but pins DNS to the address the SSRF + /// guard validated for the request URL so the connect cannot rebind to an + /// internal IP after the check (TOCTOU). The OAuth DCR/discovery/token + /// requests target author-controlled URLs and carry secrets, so they must + /// go through this rather than the unpinned client. `apply_dns_pinning` + /// lives on windmill-common's reqwest, which this crate resolves at a + /// different version (via rmcp), so pin directly with the std-typed + /// host/addrs. Empty `addrs` (IP literal or ALLOW_PRIVATE_MCP_SERVER_URLS) + /// leaves resolution untouched. + pub fn no_redirect_http_client_pinned( + target: &windmill_common::ssrf::ValidatedTarget, + ) -> Result { + let mut builder = reqwest::Client::builder() + .timeout(DEFAULT_OAUTH_HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()); + if !target.addrs.is_empty() { + builder = builder.resolve_to_addrs(&target.host, &target.addrs); + } + builder.build() + } + // Re-export oauth2 types needed for MCP OAuth flow pub use oauth2::{ basic::BasicClient, AuthUrl, ClientId, ClientSecret, CsrfToken, PkceCodeChallenge, diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index 96a2bed8df..e42c0dacfc 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -16,6 +16,24 @@ use crate::server::endpoints::EndpointTool; /// Result type for backend operations using rmcp's ErrorData directly pub type BackendResult = Result; +/// How a script/flow listing is narrowed by path at the SQL layer, *before* the +/// `ITEMS_FETCH_MAX_LIMIT` cap applies. +/// +/// This must be pushed into the query, not applied in Rust after the fetch: the +/// listing is capped to the newest N rows, so a granular token whose in-scope +/// items are not among those N would have them truncated away before any Rust +/// filter ran (returning zero tools even though the items exist). +#[derive(Debug, Clone, Copy)] +pub enum PathFilter<'a> { + /// Raw `LIKE '{prefix}%'` prefix — used to resolve a hashed tool name back to + /// its full path. + Prefix(&'a str), + /// MCP scope patterns (`*`, an exact path, or an `x/*` subtree). Mirrors + /// `is_resource_allowed`: a `*` pattern matches everything, an empty list + /// matches nothing. + Patterns(&'a [String]), +} + /// Authentication context required by the MCP server pub trait McpAuth: Send + Sync + Clone + 'static { /// Get the username @@ -62,22 +80,22 @@ pub trait McpBackend: Send + Sync + Clone + 'static { // Listing Operations // ───────────────────────────────────────────────────────────────── - /// List scripts, optionally filtered to favorites only and/or by path prefix + /// List scripts, optionally filtered to favorites only and/or by path async fn list_scripts( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, - path_prefix: Option<&str>, + path_filter: Option>, ) -> BackendResult>; - /// List flows, optionally filtered to favorites only and/or by path prefix + /// List flows, optionally filtered to favorites only and/or by path async fn list_flows( &self, auth: &Self::Auth, workspace_id: &str, favorites_only: bool, - path_prefix: Option<&str>, + path_filter: Option>, ) -> BackendResult>; /// List resource types in workspace diff --git a/backend/windmill-mcp/src/server/endpoints.rs b/backend/windmill-mcp/src/server/endpoints.rs index c3a50f9715..9e3adffc0b 100644 --- a/backend/windmill-mcp/src/server/endpoints.rs +++ b/backend/windmill-mcp/src/server/endpoints.rs @@ -21,7 +21,6 @@ pub struct EndpointTool { pub path_params_schema: Option, pub query_params_schema: Option, pub body_schema: Option, - pub path_field_renames: Option, pub query_field_renames: Option, pub body_field_renames: Option, } @@ -215,7 +214,6 @@ mod tests { "required": [] })), body_schema: None, - path_field_renames: None, query_field_renames: None, body_field_renames: None, } diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index d5d49bc746..da7032418b 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -12,7 +12,7 @@ pub mod tools; // Re-export main types pub use crate::common::types::{McpToken, MultiWorkspaceMcp, WorkspaceInfo}; -pub use backend::{BackendResult, McpAuth, McpBackend}; +pub use backend::{BackendResult, McpAuth, McpBackend, PathFilter}; pub use endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, is_endpoint_read_only, list_workspaces_tool, EndpointTool, diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index dd810269d9..f9d8c5d876 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -10,9 +10,9 @@ use crate::common::transform::{ reverse_transform, reverse_transform_key, }; use crate::common::types::{McpToken, MultiWorkspaceMcp, ResourceInfo, ToolableItem, WorkspaceId}; -use crate::server::backend::{McpAuth, McpBackend}; +use crate::server::backend::{McpAuth, McpBackend, PathFilter}; use crate::server::endpoints::{ - endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, + endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; use crate::server::tools::create_tool_from_item; use rmcp::handler::server::ServerHandler; @@ -126,20 +126,176 @@ impl Runner { } } -/// The run-by-path endpoint tools execute an arbitrary script/flow named by a -/// `path` argument. In multi-workspace mode they are the only way to run -/// scripts/flows, so their authorization must honor the `mcp:scripts:` / -/// `mcp:flows:` path scopes (not the generic endpoint scope) — otherwise a -/// granular token could run items outside its allowed paths. Returns the scope -/// resource type ("script"/"flow") for these endpoints, `None` otherwise. -fn run_by_path_scope_kind(endpoint_name: &str) -> Option<&'static str> { +/// How an endpoint tool interacts with the token's `mcp:scripts:` / `mcp:flows:` +/// path scopes. +enum EndpointPathPolicy { + /// Executes the script/flow named by the `path` argument. Gated by the + /// script/flow scope alone (an endpoint scope is not enough to run things): + /// in multi-workspace mode these are the only way to run scripts/flows, so a + /// granular token must not run items outside its allowed paths. + RunByPath(&'static str), + /// Reads/writes the script/flow named by the listed path arguments. The + /// endpoint scope grants the capability; when the token also carries path + /// patterns for `kind`, every listed argument must match them. + PathArgs { kind: &'static str, fields: &'static [&'static str] }, + /// Affects scripts without taking a checkable path (delete-by-hash) or + /// executes arbitrary code (preview). Unavailable to path-confined tokens — + /// allowing these would bypass the path patterns entirely. + Unconfinable(&'static str), +} + +fn endpoint_path_policy(endpoint_name: &str) -> Option { + use EndpointPathPolicy::*; match endpoint_name { - "runScriptByPath" => Some("script"), - "runFlowByPath" => Some("flow"), + "runScriptByPath" => Some(RunByPath("script")), + "runFlowByPath" => Some(RunByPath("flow")), + "getScriptByPath" | "deleteScriptByPath" | "createScript" => { + Some(PathArgs { kind: "script", fields: &["path"] }) + } + "getFlowByPath" | "deleteFlowByPath" | "createFlow" => { + Some(PathArgs { kind: "flow", fields: &["path"] }) + } + // updateFlow addresses the flow via the URL path and can move it to the + // path given in the body — both must stay within scope. + "updateFlow" => Some(PathArgs { kind: "flow", fields: &["path__path", "path__body"] }), + "deleteScriptByHash" | "runScriptPreviewAndWaitResult" => Some(Unconfinable("script")), _ => None, } } +/// Whether the token restricts `kind` ("script"/"flow") to specific paths. A +/// `*` pattern grants every path (see `is_resource_allowed`), so it does not +/// count as confinement. +fn path_confined(scope_config: &crate::common::scope::McpScopeConfig, kind: &str) -> bool { + if scope_config.all { + return false; + } + let patterns = match kind { + "script" => &scope_config.scripts, + "flow" => &scope_config.flows, + _ => return false, + }; + !patterns.is_empty() && !patterns.iter().any(|p| p == "*") +} + +fn require_path_arg<'a>( + endpoint_tool: &EndpointTool, + args: &'a Value, + field: &str, +) -> Result<&'a str, ErrorData> { + args.get(field) + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + ErrorData::invalid_params( + format!( + "Missing required '{}' argument for tool '{}'.", + field, endpoint_tool.name + ), + None, + ) + }) +} + +/// Whether an endpoint tool is in scope for *listing*. Run-by-path endpoints are +/// gated by the script/flow scope (`has_any` — the token can run at least one +/// path of that kind); unconfinable endpoints are hidden from path-confined +/// tokens (their calls would always be denied); every other endpoint by the +/// endpoint-name scope. +fn endpoint_tool_in_scope( + scope_config: &crate::common::scope::McpScopeConfig, + endpoint_tool: &EndpointTool, +) -> bool { + let endpoint_allowed = + !scope_config.granular || scope_config.is_allowed("endpoint", &endpoint_tool.name); + match endpoint_path_policy(&endpoint_tool.name) { + Some(EndpointPathPolicy::RunByPath(kind)) => scope_config.has_any(kind), + Some(EndpointPathPolicy::Unconfinable(kind)) => { + endpoint_allowed && !path_confined(scope_config, kind) + } + _ => endpoint_allowed, + } +} + +/// Authorize an endpoint-tool *call* against the token's MCP scopes and +/// read-only flag. Shared by single- and multi-workspace modes so both enforce +/// the same rules — otherwise a granular token could run items outside its +/// allowed paths through the single-workspace path. Run-by-path endpoints +/// (runScriptByPath/runFlowByPath) are gated by the script/flow scope for the +/// requested `path` (the endpoint-name scope alone is insufficient); other +/// endpoints by the endpoint-name scope, with script/flow path arguments +/// additionally confined to the token's path patterns when it has any; and +/// non-GET endpoints are refused for read-only tokens. +fn authorize_endpoint_call( + scope_config: &crate::common::scope::McpScopeConfig, + endpoint_tool: &EndpointTool, + args: &Value, + read_only: bool, +) -> Result<(), ErrorData> { + match endpoint_path_policy(&endpoint_tool.name) { + Some(EndpointPathPolicy::RunByPath(kind)) => { + let path = require_path_arg(endpoint_tool, args, "path")?; + // No `granular` gate: is_allowed already encodes every mode — true for + // mcp:all, pattern-matched for granular scopes, and false for + // mcp:favorites (a favorites token can't run an arbitrary path). + if !scope_config.is_allowed(kind, path) { + return Err(ErrorData::internal_error( + format!("Access denied: {} '{}' not in token scope", kind, path), + None, + )); + } + } + policy => { + if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' not in token scope", + endpoint_tool.name + ), + None, + )); + } + match policy { + Some(EndpointPathPolicy::PathArgs { kind, fields }) + if path_confined(scope_config, kind) => + { + for field in fields { + let path = require_path_arg(endpoint_tool, args, field)?; + if !scope_config.is_allowed(kind, path) { + return Err(ErrorData::internal_error( + format!("Access denied: {} '{}' not in token scope", kind, path), + None, + )); + } + } + } + Some(EndpointPathPolicy::Unconfinable(kind)) + if path_confined(scope_config, kind) => + { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' is not available to a token restricted to specific {} paths", + endpoint_tool.name, kind + ), + None, + )); + } + _ => {} + } + } + } + if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", + endpoint_tool.name + ), + None, + )); + } + Ok(()) +} + fn find_matching_path(candidates: Vec, request_name: &str) -> Option { candidates .into_iter() @@ -273,11 +429,23 @@ impl Runner { // mutating action), so skip the script/flow/hub/resource fetches // entirely — they would only be discarded below. if !read_only { + // For granular tokens, push the scope patterns into the SQL query so + // in-scope items survive the fetch cap (see `PathFilter`). The Rust + // filter below still runs as a defense-in-depth check. Non-granular + // (favorites/all) tokens are already narrowed by the favorites join + // or intentionally unfiltered. + let script_filter = scope_config + .granular + .then(|| PathFilter::Patterns(scope_config.scripts.as_slice())); + let flow_filter = scope_config + .granular + .then(|| PathFilter::Patterns(scope_config.flows.as_slice())); + let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!( self.backend - .list_scripts(auth, workspace_id, favorites_only, None), + .list_scripts(auth, workspace_id, favorites_only, script_filter), self.backend - .list_flows(auth, workspace_id, favorites_only, None), + .list_flows(auth, workspace_id, favorites_only, flow_filter), self.backend.list_resource_types(auth, workspace_id), async { if let Some(ref apps) = scope_config.hub_apps { @@ -361,10 +529,13 @@ impl Runner { } } - // Add endpoint tools from the generated MCP tools, filtered by scope + // Add endpoint tools from the generated MCP tools, filtered by scope. + // Uses the same run-by-path-aware gate as multi-workspace mode so a + // granular token only sees runScriptByPath / runFlowByPath when it can + // actually run scripts / flows. let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in endpoint_tools { - if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) { + if !endpoint_tool_in_scope(scope_config, &endpoint_tool) { continue; } if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) { @@ -391,27 +562,9 @@ impl Runner { let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in &endpoint_tools { if endpoint_tool.name.as_ref() == name.as_ref() { - // Validate endpoint scope - if scope_config.granular - && !scope_config.is_allowed("endpoint", &endpoint_tool.name) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", - endpoint_tool.name - ), - None, - )); - } + // Authorize against the token's MCP scopes and read-only flag, + // including the run-by-path path check (shared with multi mode). + authorize_endpoint_call(scope_config, endpoint_tool, &args, read_only)?; // This is an endpoint tool, call via backend let result = self @@ -456,11 +609,12 @@ impl Runner { (type_str, version_id, true) } else { let path_prefix = extract_path_prefix_from_hashed(name.as_ref()); + let path_filter = path_prefix.as_deref().map(PathFilter::Prefix); let favorites_only = scope_config.favorites; let matched_path = if type_str == "script" { find_matching_path( self.backend - .list_scripts(auth, workspace_id, favorites_only, path_prefix.as_deref()) + .list_scripts(auth, workspace_id, favorites_only, path_filter) .await .map_err(|e| ErrorData::internal_error(e.message, None))?, name.as_ref(), @@ -468,7 +622,7 @@ impl Runner { } else { find_matching_path( self.backend - .list_flows(auth, workspace_id, favorites_only, path_prefix.as_deref()) + .list_flows(auth, workspace_id, favorites_only, path_filter) .await .map_err(|e| ErrorData::internal_error(e.message, None))?, name.as_ref(), @@ -567,16 +721,7 @@ impl Runner { let endpoint_tools = self.backend.all_endpoint_tools(); for endpoint_tool in endpoint_tools { - // Run-by-path tools are gated by script/flow scope (they run an - // arbitrary path); every other endpoint by the endpoint scope. - let allowed = match run_by_path_scope_kind(&endpoint_tool.name) { - Some(kind) => scope_config.has_any(kind), - None => { - !scope_config.granular - || scope_config.is_allowed("endpoint", &endpoint_tool.name) - } - }; - if !allowed { + if !endpoint_tool_in_scope(scope_config, &endpoint_tool) { continue; } if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) { @@ -628,59 +773,13 @@ impl Runner { ) })?; - // Authorize the tool. Run-by-path endpoints (runScriptByPath / - // runFlowByPath) run an arbitrary `path` and must be checked against the - // script/flow scope for that path — the endpoint scope alone would let a - // granular token run items outside its allowed paths. - match run_by_path_scope_kind(&endpoint_tool.name) { - Some(kind) => { - let path = args - .get("path") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - ErrorData::invalid_params( - format!( - "Missing required 'path' argument for tool '{}'.", - endpoint_tool.name - ), - None, - ) - })?; - // No `granular` gate: is_allowed already encodes every mode — - // true for mcp:all, pattern-matched for granular scopes, and - // false for mcp:favorites (a favorites token can't run an - // arbitrary path, only its enumerated favorites). - if !scope_config.is_allowed(kind, path) { - return Err(ErrorData::internal_error( - format!("Access denied: {} '{}' not in token scope", kind, path), - None, - )); - } - } - None => { - if scope_config.granular - && !scope_config.is_allowed("endpoint", &endpoint_tool.name) - { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' not in token scope", - endpoint_tool.name - ), - None, - )); - } - } - } - if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) { - return Err(ErrorData::internal_error( - format!( - "Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations", - endpoint_tool.name - ), - None, - )); - } + // Authorize the tool against the token's MCP scopes and read-only flag + // (shared with single-workspace mode). Run-by-path endpoints + // (runScriptByPath / runFlowByPath) run an arbitrary `path` and are + // checked against the script/flow scope for that path — the endpoint + // scope alone would let a granular token run items outside its allowed + // paths. + authorize_endpoint_call(scope_config, endpoint_tool, &args, read_only)?; // Workspace-scoped endpoints need an explicit target workspace and a // per-workspace auth; global endpoints (e.g. docs) use the base identity. @@ -733,3 +832,270 @@ impl Runner { )])) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::common::scope::{parse_mcp_scopes, McpScopeConfig}; + use serde_json::json; + use std::borrow::Cow; + + fn cfg(scopes: &[&str]) -> McpScopeConfig { + parse_mcp_scopes(&scopes.iter().map(|s| s.to_string()).collect::>()).unwrap() + } + + fn ep(name: &'static str, method: &'static str) -> EndpointTool { + EndpointTool { + name: Cow::Borrowed(name), + description: Cow::Borrowed(""), + instructions: Cow::Borrowed(""), + path: Cow::Borrowed("/w/{workspace}/jobs/run/p/{path}"), + method: Cow::Borrowed(method), + path_params_schema: None, + query_params_schema: None, + body_schema: None, + query_field_renames: None, + body_field_renames: None, + } + } + + // The core invariant: a folder-scoped token (mcp:endpoints:* + + // mcp:scripts:f/team/*, as the folder-scope UI emits) must not run a script + // outside its allowed folders via runScriptByPath — mcp:endpoints:* alone + // must never authorize an arbitrary path. + #[test] + fn run_by_path_call_enforces_script_scope() { + let config = cfg(&[ + "mcp:scripts:f/team/*", + "mcp:flows:f/team/*", + "mcp:endpoints:*", + ]); + let tool = ep("runScriptByPath", "POST"); + + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/deploy"}), false) + .is_ok() + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/admin"}), false) + .is_err() + ); + } + + #[test] + fn run_by_path_call_requires_path_arg() { + let tool = ep("runFlowByPath", "POST"); + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &tool, &json!({}), false).is_err()); + } + + #[test] + fn run_by_path_flow_scope_independent_of_script_scope() { + // A flow-only token can run flows by path but not scripts by path. + let config = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &config, + &ep("runFlowByPath", "POST"), + &json!({"path": "f/team/x"}), + false + ) + .is_ok()); + assert!(authorize_endpoint_call( + &config, + &ep("runScriptByPath", "POST"), + &json!({"path": "f/team/x"}), + false + ) + .is_err()); + } + + #[test] + fn non_run_by_path_gated_by_endpoint_scope() { + let get_var = ep("getVariable", "GET"); + assert!(authorize_endpoint_call( + &cfg(&["mcp:endpoints:getVariable"]), + &get_var, + &json!({"path": "u/a/b"}), + false + ) + .is_ok()); + // A granular token without the endpoint scope is denied. + assert!(authorize_endpoint_call( + &cfg(&["mcp:scripts:f/team/*"]), + &get_var, + &json!({"path": "u/a/b"}), + false + ) + .is_err()); + // mcp:all (non-granular) allows any endpoint. + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &get_var, &json!({}), false).is_ok()); + } + + #[test] + fn read_only_refuses_non_get_endpoint() { + // Reaches the read-only check via mcp:all so scope isn't the blocker. + assert!(authorize_endpoint_call( + &cfg(&["mcp:all"]), + &ep("createResource", "POST"), + &json!({}), + true + ) + .is_err()); + assert!(authorize_endpoint_call( + &cfg(&["mcp:all"]), + &ep("getVariable", "GET"), + &json!({"path": "u/a/b"}), + true + ) + .is_ok()); + } + + #[test] + fn listing_run_by_path_needs_runnable_scope() { + let tool = ep("runScriptByPath", "POST"); + // An endpoint-only token cannot run any script, so the tool isn't listed. + assert!(!endpoint_tool_in_scope(&cfg(&["mcp:endpoints:*"]), &tool)); + // A script-scoped token can, so it is listed. + assert!(endpoint_tool_in_scope( + &cfg(&["mcp:scripts:f/team/*"]), + &tool + )); + assert!(endpoint_tool_in_scope(&cfg(&["mcp:all"]), &tool)); + // Non-run-by-path endpoints are governed by the endpoint scope. + let get_var = ep("getVariable", "GET"); + assert!(endpoint_tool_in_scope( + &cfg(&["mcp:endpoints:getVariable"]), + &get_var + )); + assert!(!endpoint_tool_in_scope( + &cfg(&["mcp:scripts:f/team/*"]), + &get_var + )); + } + + // A folder-scoped token must not read/write/delete scripts or flows outside + // its allowed paths through the non-run endpoint tools either. + #[test] + fn path_arg_tools_confined_by_path_patterns() { + let config = cfg(&[ + "mcp:scripts:f/team/*", + "mcp:flows:f/team/*", + "mcp:endpoints:*", + ]); + for name in ["getScriptByPath", "deleteScriptByPath", "createScript"] { + let tool = ep(name, "POST"); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/x"}), false) + .is_ok(), + "{name} should allow in-scope path" + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/x"}), false) + .is_err(), + "{name} should deny out-of-scope path" + ); + // Confinement can't be verified without the path argument. + assert!( + authorize_endpoint_call(&config, &tool, &json!({}), false).is_err(), + "{name} should require the path argument when confined" + ); + } + for name in ["getFlowByPath", "deleteFlowByPath", "createFlow"] { + let tool = ep(name, "POST"); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/team/x"}), false) + .is_ok(), + "{name} should allow in-scope path" + ); + assert!( + authorize_endpoint_call(&config, &tool, &json!({"path": "f/secret/x"}), false) + .is_err(), + "{name} should deny out-of-scope path" + ); + } + } + + // A token that never expressed path patterns (endpoints-only) is not + // confined: the endpoint scope alone authorizes any path. + #[test] + fn path_arg_tools_unconfined_without_path_patterns() { + let config = cfg(&["mcp:endpoints:*"]); + for name in ["getScriptByPath", "createScript", "deleteFlowByPath"] { + assert!(authorize_endpoint_call( + &config, + &ep(name, "POST"), + &json!({"path": "f/anywhere/x"}), + false + ) + .is_ok()); + } + // A `*` pattern grants every path, so it doesn't confine either. + let star = cfg(&["mcp:scripts:*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &star, + &ep("createScript", "POST"), + &json!({"path": "f/anywhere/x"}), + false + ) + .is_ok()); + } + + // updateFlow both addresses a flow (URL path) and can move it (body path): + // a confined token must have both within scope. + #[test] + fn update_flow_checks_target_and_destination_paths() { + let config = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + let tool = ep("updateFlow", "POST"); + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/team/a", "path__body": "f/team/b"}), + false + ) + .is_ok()); + // Moving a flow out of the allowed folder is denied. + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/team/a", "path__body": "f/secret/a"}), + false + ) + .is_err()); + // Touching a flow outside the allowed folder is denied. + assert!(authorize_endpoint_call( + &config, + &tool, + &json!({"path__path": "f/secret/a", "path__body": "f/team/a"}), + false + ) + .is_err()); + } + + // Tools that can't be path-checked (delete-by-hash) or execute arbitrary + // code (preview) would bypass path confinement, so a path-confined token is + // denied them entirely — and doesn't see them listed. + #[test] + fn unconfinable_tools_denied_for_path_confined_token() { + let confined = cfg(&["mcp:scripts:f/team/*", "mcp:endpoints:*"]); + for name in ["deleteScriptByHash", "runScriptPreviewAndWaitResult"] { + let tool = ep(name, "POST"); + assert!(authorize_endpoint_call(&confined, &tool, &json!({}), false).is_err()); + assert!(!endpoint_tool_in_scope(&confined, &tool)); + // Without script path patterns the tools stay available. + assert!( + authorize_endpoint_call(&cfg(&["mcp:endpoints:*"]), &tool, &json!({}), false) + .is_ok() + ); + assert!(authorize_endpoint_call(&cfg(&["mcp:all"]), &tool, &json!({}), false).is_ok()); + assert!(endpoint_tool_in_scope(&cfg(&["mcp:endpoints:*"]), &tool)); + } + // Flow-only confinement doesn't affect script-kind unconfinable tools. + let flow_confined = cfg(&["mcp:flows:f/team/*", "mcp:endpoints:*"]); + assert!(authorize_endpoint_call( + &flow_confined, + &ep("deleteScriptByHash", "POST"), + &json!({}), + false + ) + .is_ok()); + } +} diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 553eb69917..9121f6183d 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -361,12 +361,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result>, +} + +#[cfg(feature = "parquet")] +impl AmbientAwsCredentials { + // Credentials without an expiry (env vars, static profile) are still re-resolved + // periodically so runtime changes to the environment are eventually picked up. + const NO_EXPIRY_TTL: std::time::Duration = std::time::Duration::from_secs(300); + const EXPIRY_MARGIN: std::time::Duration = std::time::Duration::from_secs(120); + + fn still_valid(creds: &aws_sdk_sts::config::Credentials, age: std::time::Duration) -> bool { + match creds.expiry() { + Some(expiry) => std::time::SystemTime::now() + Self::EXPIRY_MARGIN < expiry, + None => age < Self::NO_EXPIRY_TTL, + } + } + + async fn get(&self) -> anyhow::Result { + if let Some((creds, fetched_at)) = self.cached.read().await.as_ref() { + if Self::still_valid(creds, fetched_at.elapsed()) { + return Ok(creds.clone()); + } + } + // The write lock is held across the chain resolution so concurrent requests don't all + // hit the metadata service at once. + let mut guard = self.cached.write().await; + if let Some((creds, fetched_at)) = guard.as_ref() { + if Self::still_valid(creds, fetched_at.elapsed()) { + return Ok(creds.clone()); + } + } + let creds = self.chain.provide_credentials().await.map_err(|e| { + anyhow::anyhow!( + "no S3 access key/secret key is configured and no ambient AWS credentials could \ + be loaded through the AWS SDK default chain (env vars, profile, ECS/EC2 instance \ + role): {cause}. If an EC2/ECS instance role is expected to be used, the instance \ + metadata service must be reachable from the process running Windmill — on EC2 the \ + AWS Rust SDK only supports IMDSv2, so when Windmill runs in a Docker container \ + the instance metadata hop limit (HttpPutResponseHopLimit) must be at least 2", + cause = format!("{:#}", anyhow::Error::new(e)) + ) + })?; + *guard = Some((creds.clone(), std::time::Instant::now())); + Ok(creds) + } +} + +#[cfg(feature = "parquet")] +lazy_static::lazy_static! { + static ref AMBIENT_AWS_CREDS_PROVIDERS: Cache> = + Cache::new(20); +} + +#[cfg(feature = "parquet")] +async fn ambient_aws_credentials_provider(region: &str) -> Arc { + // Single-flight: concurrent cold misses for the same region must share one provider, + // otherwise each gets its own instance and their per-instance refresh locks can't serialize + // the initial credential resolution — every caller would hit the metadata service. + match AMBIENT_AWS_CREDS_PROVIDERS + .get_value_or_guard_async(region) + .await + { + Ok(provider) => provider, + Err(guard) => { + let chain = DefaultCredentialsChain::builder() + .region(Region::new(region.to_string())) + .build() + .await; + let provider = Arc::new(AmbientAwsCredentials { chain, cached: RwLock::new(None) }); + let _ = guard.insert(provider.clone()); + provider + } + } +} + #[cfg(feature = "parquet")] #[derive(Debug)] struct AwsCredentialAdapter { - pub inner: DefaultCredentialsChain, + pub inner: Arc, } #[cfg(feature = "parquet")] @@ -775,9 +852,9 @@ struct AwsCredentialAdapter { impl CredentialProvider for AwsCredentialAdapter { type Credential = AwsCredential; async fn get_credential(&self) -> object_store::Result> { - let creds = self.inner.provide_credentials().await.map_err(|e| { - tracing::error!("Error getting credentials: {:?}", e); - object_store::Error::Generic { store: "AWS", source: Box::new(e) } + let creds = self.inner.get().await.map_err(|e| { + tracing::error!("Error getting AWS credentials: {e:#}"); + object_store::Error::Generic { store: "AWS", source: e.into() } })?; Ok(Arc::new(Self::Credential { key_id: creds.access_key_id().to_string(), @@ -1481,6 +1558,45 @@ pub async fn get_logs_from_store( mod tests { use super::*; + // --- ambient credentials cache tests --- + + #[cfg(feature = "parquet")] + #[test] + fn test_ambient_credentials_still_valid() { + use std::time::{Duration, SystemTime}; + + fn creds(expiry: Option) -> aws_sdk_sts::config::Credentials { + let mut builder = aws_sdk_sts::config::Credentials::builder() + .access_key_id("AK") + .secret_access_key("SK") + .provider_name("test"); + if let Some(expiry) = expiry { + builder = builder.expiry(expiry); + } + builder.build() + } + + // Expiry far in the future: valid regardless of fetch time + assert!(AmbientAwsCredentials::still_valid( + &creds(Some(SystemTime::now() + Duration::from_secs(3600))), + Duration::ZERO + )); + // Expiry within the refresh margin: must be re-fetched + assert!(!AmbientAwsCredentials::still_valid( + &creds(Some(SystemTime::now() + Duration::from_secs(30))), + Duration::ZERO + )); + // No expiry: valid while fresh, re-fetched after the TTL + assert!(AmbientAwsCredentials::still_valid( + &creds(None), + Duration::ZERO + )); + assert!(!AmbientAwsCredentials::still_valid( + &creds(None), + AmbientAwsCredentials::NO_EXPIRY_TTL + Duration::from_secs(1) + )); + } + // --- render_endpoint tests --- #[test] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 4ce45eb8c0..f2d0425710 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -25,6 +25,7 @@ use serde::{ser::SerializeMap, Serialize}; use serde_json::{json, value::RawValue}; use sqlx::{types::Json, Acquire, Pool, Postgres, Transaction}; use sqlx::{Encode, PgExecutor}; +use std::borrow::Cow; use tokio::sync::mpsc::Sender; use tokio::sync::oneshot; use tokio::task::JoinHandle; @@ -49,7 +50,7 @@ use windmill_common::runnable_settings::{ RunnableSettings, RunnableSettingsTrait, }; use windmill_common::triggers::TriggerMetadata; -use windmill_common::utils::{calculate_hash, configure_client, now_from_db}; +use windmill_common::utils::{calculate_hash, configure_client, now_from_db, strip_json_nul}; use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; use windmill_common::otel_oss::{ @@ -634,6 +635,12 @@ pub trait ValidableJson { fn wm_failure(&self) -> Option; fn result_metadata(&self) -> ResultMetadata; fn size(&self) -> usize; + /// The result as JSON text, for binding into the `jsonb` `result` column. + /// `Box` is already serialized and returns a zero-cost borrow; + /// other impls serialize on demand. Callers pass this through + /// `strip_json_nul` before the INSERT, since a genuine NUL escape would + /// abort the write with 22P05. + fn serialized_json(&self) -> Cow<'_, str>; } /// The Windmill-specific markers we look for inside a job's result. @@ -692,6 +699,10 @@ impl ValidableJson for WrappedError { fn size(&self) -> usize { 0 } + + fn serialized_json(&self) -> Cow<'_, str> { + Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string())) + } } impl ValidableJson for Box { @@ -714,6 +725,11 @@ impl ValidableJson for Box { fn size(&self) -> usize { self.get().len() } + + fn serialized_json(&self) -> Cow<'_, str> { + // Already serialized JSON text — borrow it, no re-serialization. + Cow::Borrowed(self.get()) + } } impl ValidableJson for Arc { @@ -736,6 +752,10 @@ impl ValidableJson for Arc { fn size(&self) -> usize { T::size(&self) } + + fn serialized_json(&self) -> Cow<'_, str> { + T::serialized_json(&self) + } } impl ValidableJson for serde_json::Value { @@ -758,6 +778,10 @@ impl ValidableJson for serde_json::Value { fn size(&self) -> usize { self.size_hint() } + + fn serialized_json(&self) -> Cow<'_, str> { + Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string())) + } } impl ValidableJson for Json { @@ -780,6 +804,10 @@ impl ValidableJson for Json { fn size(&self) -> usize { self.0.size() } + + fn serialized_json(&self) -> Cow<'_, str> { + self.0.serialized_json() + } } pub async fn register_metric( @@ -925,8 +953,13 @@ lazy_static::lazy_static! { static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), (bool, Option)> = Cache::new(10000); // Cache for workspace error handler settings with 60s TTL - // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, expiry_timestamp) - static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, bool, i64)> = Cache::new(1000); + // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, report_to_instance_alerts, expiry_timestamp) + static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache, Option>>, bool, bool, bool, i64)> = Cache::new(1000); + + // Best-effort per-worker throttle for the instance-channel fallback: a flapping runnable + // would otherwise turn every failure into outbound Slack/SMTP traffic on channels shared by + // the whole instance. Key: workspace_id, Value: (last_sent_epoch, failures suppressed since) + static ref INSTANCE_ALERT_THROTTLE: Cache = Cache::new(1000); // Cache for workspace success handler settings with 60s TTL // Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp) @@ -934,6 +967,7 @@ lazy_static::lazy_static! { } const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60; +const INSTANCE_ALERT_COOLDOWN_SECONDS: i64 = 60; pub async fn add_completed_job( db: &Pool, @@ -1079,15 +1113,24 @@ async fn commit_completed_job( // Resolve the concurrency-limit settings on the pool *before* opening the // completion transaction: doing it inside the tx would hold a second // simultaneous connection from the small per-worker pool. - let has_concurrent_limit = completed_job.concurrent_limit.is_some() - || windmill_common::runnable_settings::prefetch_cached_from_handle( - completed_job.runnable_settings_handle, - db, - ) - .await? - .1 - .concurrent_limit - .is_some(); + let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit) + || has_active_concurrency_limit( + windmill_common::runnable_settings::prefetch_cached_from_handle( + completed_job.runnable_settings_handle, + db, + ) + .await? + .1 + .concurrent_limit, + ); + + // A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that + // the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape + // sequence"), which would abort the whole completion INSERT. Strip it before + // binding — near-zero cost when clean: a single scan, and for an + // already-serialized `RawValue` result the serialization itself is a borrow. + let serialized_result = result.serialized_json(); + let sanitized_result = strip_json_nul(serialized_result.as_ref()); let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1107,7 +1150,7 @@ async fn commit_completed_job( , status , worker ) - SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3, $10, $5, $6, + SELECT q.workspace_id, q.id, started_at, COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000), $3::text::jsonb, $10, $5, $6, flow_status, workflow_as_code_status, $8, CASE WHEN $4::BOOL THEN 'canceled'::job_status WHEN $7::BOOL THEN 'skipped'::job_status @@ -1115,10 +1158,10 @@ async fn commit_completed_job( ELSE 'failure'::job_status END AS status, q.worker FROM v2_job_queue q LEFT JOIN v2_job_status USING (id) WHERE q.id = $1 - ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3 RETURNING duration_ms AS \"duration_ms!\"", + ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb RETURNING duration_ms AS \"duration_ms!\"", /* $1 */ completed_job.id, /* $2 */ success, - /* $3 */ result as Json<&T>, + /* $3 */ sanitized_result.as_ref(), /* $4 */ canceled_by.is_some(), /* $5 */ canceled_by.clone().map(|cb| cb.username).flatten(), /* $6 */ canceled_by.clone().map(|cb| cb.reason).flatten(), @@ -1156,7 +1199,14 @@ async fn commit_completed_job( } }; - if let Some(labels) = result.wm_labels() { + if let Some(mut labels) = result.wm_labels() { + // A `\u0000` inside a wm_labels entry decodes to a real NUL that the + // `text[]` column rejects, which would abort this same transaction (and + // roll back the sanitized result insert) exactly like an unsanitized + // result. Strip it so the labels match the sanitized result. + for label in &mut labels { + label.retain(|c| c != '\0'); + } sqlx::query!( "UPDATE v2_job SET labels = ( SELECT array_agg(DISTINCT all_labels) @@ -2081,7 +2131,7 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( async fn fetch_error_handler_from_db( db: &Pool, w_id: &str, -) -> Result<(Option, Option>>, bool, bool), Error> { +) -> Result<(Option, Option>>, bool, bool, bool), Error> { sqlx::query_as::< _, ( @@ -2089,6 +2139,7 @@ async fn fetch_error_handler_from_db( Option>>, Option, Option, + bool, ), >( r#" @@ -2096,23 +2147,28 @@ async fn fetch_error_handler_from_db( error_handler->>'path', (error_handler->'extra_args')::text::json, (error_handler->>'muted_on_cancel')::boolean, - (error_handler->>'muted_on_user_path')::boolean - FROM workspace_settings - WHERE workspace_id = $1 + (error_handler->>'muted_on_user_path')::boolean, + ws.error_handler_fallback_to_instance_alerts AND w.parent_workspace_id IS NULL + FROM workspace_settings ws + JOIN workspace w ON w.id = ws.workspace_id + WHERE ws.workspace_id = $1 "#, ) .bind(w_id) .fetch_optional(db) .await .context("fetching error handler info from workspace_settings")? - .map(|(path, extra_args, muted_on_cancel, muted_on_user_path)| { - ( - path, - extra_args, - muted_on_cancel.unwrap_or(false), - muted_on_user_path.unwrap_or(false), - ) - }) + .map( + |(path, extra_args, muted_on_cancel, muted_on_user_path, report_to_instance_alerts)| { + ( + path, + extra_args, + muted_on_cancel.unwrap_or(false), + muted_on_user_path.unwrap_or(false), + report_to_instance_alerts, + ) + }, + ) .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}"))) } @@ -2130,15 +2186,22 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, + report_to_instance_alerts, ) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) { - if cached.4 > now { - (cached.0.clone(), cached.1.clone(), cached.2, cached.3) + if cached.5 > now { + ( + cached.0.clone(), + cached.1.clone(), + cached.2, + cached.3, + cached.4, + ) } else { let row = fetch_error_handler_from_db(db, w_id).await?; let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row.0.clone(), row.1.clone(), row.2, row.3, expiry), + (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry), ); row } @@ -2147,11 +2210,17 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row.0.clone(), row.1.clone(), row.2, row.3, expiry), + (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry), ); row }; + // Nothing to do for the vast majority of workspaces, and returning here keeps the + // per-runnable mute lookup below off the path of every failed job. + if error_handler.is_none() && !report_to_instance_alerts { + return Ok(()); + } + if is_canceled && error_handler_muted_on_cancel { return Ok(()); } @@ -2165,51 +2234,90 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> } } - if let Some(error_handler) = error_handler { - let ws_error_handler_muted: Option = match queued_job.kind { - JobKind::Script => { - sqlx::query_scalar!( + let ws_error_handler_muted: Option = match queued_job.kind { + JobKind::Script => { + sqlx::query_scalar!( "SELECT ws_error_handler_muted FROM script WHERE workspace_id = $1 AND hash = $2", queued_job.workspace_id, queued_job.runnable_id.map(|x| x.0), ) - .fetch_optional(db) - .await? - } - JobKind::Flow => { - sqlx::query_scalar!( - "SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2", - queued_job.workspace_id, - queued_job.runnable_path.clone(), - ) - .fetch_optional(db) - .await? - } - _ => None, - }; - - let muted = ws_error_handler_muted.unwrap_or(false); - if !muted { - tracing::info!("workspace error handled for job {}", &queued_job.id); - - push_error_handler( - db, - queued_job.id, - queued_job.schedule_path(), + .fetch_optional(db) + .await? + } + JobKind::Flow => { + sqlx::query_scalar!( + "SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2", + queued_job.workspace_id, queued_job.runnable_path.clone(), - queued_job.is_flow(), - &queued_job.workspace_id, - &error_handler, - result, - None, - queued_job.started_at, - error_handler_extra_args, - &queued_job.permissioned_as_email, - false, - false, - None, ) - .await?; + .fetch_optional(db) + .await? + } + _ => None, + }; + + if ws_error_handler_muted.unwrap_or(false) { + return Ok(()); + } + + if let Some(error_handler) = error_handler { + tracing::info!("workspace error handled for job {}", &queued_job.id); + + push_error_handler( + db, + queued_job.id, + queued_job.schedule_path(), + queued_job.runnable_path.clone(), + queued_job.is_flow(), + &queued_job.workspace_id, + &error_handler, + result, + None, + queued_job.started_at, + error_handler_extra_args, + &queued_job.permissioned_as_email, + false, + false, + None, + ) + .await?; + } else if !is_canceled { + // A cancellation is a human action rather than an operational failure, and unlike the + // handler path this one has no per-workspace toggle to opt out of reporting them. + let suppressed = match INSTANCE_ALERT_THROTTLE.get(w_id) { + Some((last_sent, suppressed)) + if now - last_sent < INSTANCE_ALERT_COOLDOWN_SECONDS => + { + INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (last_sent, suppressed + 1)); + None + } + entry => { + INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (now, 0)); + Some(entry.map(|(_, suppressed)| suppressed).unwrap_or(0)) + } + }; + if let Some(suppressed) = suppressed { + tracing::info!( + "reporting failed job {} to the instance critical alert channels", + &queued_job.id + ); + let base_url = windmill_common::BASE_URL.load(); + let rollup = if suppressed > 0 { + format!( + " (and {suppressed} more failure(s) in the preceding {INSTANCE_ALERT_COOLDOWN_SECONDS}s)" + ) + } else { + String::new() + }; + windmill_common::utils::send_workspace_error_to_instance_channels( + format!( + "Job {} failed in workspace {w_id} ({base_url}/run/{}?workspace={w_id}){rollup}", + queued_job.runnable_path.as_deref().unwrap_or("preview"), + queued_job.id + ), + db, + ) + .await; } } Ok(()) @@ -3878,7 +3986,7 @@ pub async fn pull( let pulled_job_result = match job { #[cfg(feature = "private")] Some(job) - if concurrency_settings.concurrent_limit.is_some() + if has_active_concurrency_limit(concurrency_settings.concurrent_limit) // Concurrency limit is available for either enterprise job or dependency job && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING)) => { @@ -3942,7 +4050,8 @@ pub async fn pull( .1 .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); - let has_concurent_limit = concurrency_settings.concurrent_limit.is_some(); + let has_concurent_limit = + has_active_concurrency_limit(concurrency_settings.concurrent_limit); #[cfg(not(feature = "enterprise"))] if has_concurent_limit && !job.is_dependency() { @@ -3951,7 +4060,7 @@ pub async fn pull( #[cfg(not(feature = "enterprise"))] let has_concurent_limit = job.is_dependency() - && job.concurrent_limit.is_some() + && has_active_concurrency_limit(job.concurrent_limit) && cfg!(feature = "private") && !*WMDEBUG_NO_DEBOUNCING; // if we don't have private flag, we don't have concurrency limit @@ -4119,6 +4228,13 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( Ok(job_and_suspended) } +/// A concurrency limit is only active when it caps at 1+ slots. `Some(0)` (or negative) is +/// a disabled limit, not a zero-slot one — see [`ConcurrencySettings::normalized`]. The gate +/// checks must use this instead of `.is_some()` so a legacy stored `0` behaves as disabled. +pub fn has_active_concurrency_limit(concurrent_limit: Option) -> bool { + concurrent_limit.is_some_and(|n| n > 0) +} + pub async fn custom_concurrency_key( db: &Pool, job_id: &Uuid, @@ -5614,6 +5730,10 @@ async fn push_inner<'c, 'd>( restarted_from_val.step_id.as_str(), restarted_from_val.branch_or_iteration_n, restarted_from_val.flow_version, + restarted_from_val.nested.is_some(), + // RawFlow queues the request's (possibly edited) definition, not the + // stored one, so zombie reuse of the stored step is unsafe here. + false, ) .await?; FlowStatus { @@ -6003,6 +6123,10 @@ async fn push_inner<'c, 'd>( step_id.as_str(), branch_or_iteration_n, flow_version, + nested.is_some(), + // RestartedFlow resolves and queues the completed job's stored definition, so the + // step validated for reuse is the one that will run. + true, ) .await?; @@ -6098,6 +6222,11 @@ async fn push_inner<'c, 'd>( }, }; + // Guard against an already-stored `concurrent_limit <= 0` reaching the queue: it would + // register a zero-slot concurrency key and permanently block the job. Coerce it to + // disabled before it is persisted onto the job row / concurrency key here. + concurrency_settings = concurrency_settings.normalized(); + // Enforce concurrency limit on all dependency jobs. // TODO: We can ignore this for scripts djobs. The main reason we need all djobs to be sequential is because we have // nodes_to_relock and we need all locks whose corresponding steps aren't in nodes_to_relock be already present. @@ -6280,15 +6409,29 @@ async fn push_inner<'c, 'd>( ) .unzip(); - if concurrency_settings.concurrent_limit.is_some() { - insert_concurrency_key( + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + check_workspace_queue_cap(&mut *tx, workspace_id).await?; + } + + if has_active_concurrency_limit(concurrency_settings.concurrent_limit) { + let concurrency_key = resolve_concurrency_key( workspace_id, &args, &runnable_path, job_kind, concurrency_settings.concurrency_key.clone(), + ); + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + check_concurrency_key_queue_cap(&mut *tx, &concurrency_key).await?; + } + insert_resolved_concurrency_key( + &concurrency_key, &mut *tx, job_id, + &runnable_path, + workspace_id, ) .await?; } @@ -6617,16 +6760,14 @@ async fn push_inner<'c, 'd>( Ok((job_id, tx)) } -pub async fn insert_concurrency_key<'d, 'c>( +fn resolve_concurrency_key<'d>( workspace_id: &str, args: &PushArgs<'d>, script_path: &Option, job_kind: JobKind, custom_concurrency_key: Option, - db: impl PgExecutor<'c>, - job_id: Uuid, -) -> Result<(), Error> { - let concurrency_key = custom_concurrency_key +) -> String { + custom_concurrency_key .map(|x| { let interpolated = interpolate_args(x.clone(), args, workspace_id); // In cloud mode, enforce workspace isolation by prefixing with workspace @@ -6648,7 +6789,50 @@ pub async fn insert_concurrency_key<'d, 'c>( workspace_id, script_path.as_ref(), &job_kind, - )); + )) +} + +/// Resolves the concurrency key, applies the cloud-only queue-depth cap, then registers the key. +/// +/// `concurrent_limit: None` still registers the key but skips the cap: without a limit nothing +/// serializes the key, so there is no backlog to bound. Callers register keys for tag +/// interpolation alone, so the two are not interchangeable. +/// +/// Takes a `Copy` executor because the cap runs a query before the insert on the same one. `push` +/// cannot use this — it holds a `&mut Transaction` — so it performs the same three steps inline. +pub async fn insert_concurrency_key_capped<'d, 'c, E: PgExecutor<'c> + Copy>( + workspace_id: &str, + args: &PushArgs<'d>, + script_path: &Option, + job_kind: JobKind, + custom_concurrency_key: Option, + concurrent_limit: Option, + db: E, + job_id: Uuid, +) -> Result<(), Error> { + let concurrency_key = resolve_concurrency_key( + workspace_id, + args, + script_path, + job_kind, + custom_concurrency_key, + ); + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED && has_active_concurrency_limit(concurrent_limit) { + check_concurrency_key_queue_cap(db, &concurrency_key).await?; + } + #[cfg(not(feature = "cloud"))] + let _ = concurrent_limit; + insert_resolved_concurrency_key(&concurrency_key, db, job_id, script_path, workspace_id).await +} + +async fn insert_resolved_concurrency_key<'c>( + concurrency_key: &str, + db: impl PgExecutor<'c>, + job_id: Uuid, + script_path: &Option, + workspace_id: &str, +) -> Result<(), Error> { sqlx::query!( "WITH inserted_concurrency_counter AS ( INSERT INTO concurrency_counter (concurrency_id, job_uuids) @@ -6666,6 +6850,145 @@ pub async fn insert_concurrency_key<'d, 'c>( Ok(()) } +/// Counts jobs *waiting* behind `concurrency_key`, scanning at most `limit` rows. +/// +/// Three constraints on the query below, each pinned by a test in +/// `tests/concurrency_key_queue_depth_test.rs`: +/// - Any `scheduled_for`: the limiter parks blocked jobs in the future, so a gated backlog is +/// almost entirely future-dated and `scheduled_for <= now()` would never see it. +/// - `running = false`: running jobs keep `ended_at` NULL, and counting them would charge a key +/// for the concurrency it is licensed to use. +/// - `LIMIT` inside the `EXISTS`, not outside: rows whose job left the queue without +/// `add_completed_job` keep `ended_at` NULL forever (the retention sweep only matches +/// `ended_at <= …`), so an outside `LIMIT` would let them be rescanned on every push. +/// +/// Takes an already-resolved key and performs no authorization; `pub` only so the integration +/// test can reach it. Never call it with a caller-supplied key — it would leak queue depth +/// across workspaces. +pub async fn concurrency_key_queue_depth<'c>( + db: impl PgExecutor<'c>, + concurrency_key: &str, + limit: i64, +) -> Result { + sqlx::query_scalar!( + "SELECT count(*) FROM ( + SELECT ck.job_id FROM concurrency_key ck + WHERE ck.key = $1 AND ck.ended_at IS NULL + LIMIT $2 + ) s WHERE EXISTS ( + SELECT 1 FROM v2_job_queue q WHERE q.id = s.job_id AND q.running = false + )", + concurrency_key, + limit, + ) + .fetch_one(db) + .warn_after_seconds(3) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not count queued jobs for concurrency_key={concurrency_key}: {e:#}" + )) + }) + .map(|c| c.unwrap_or(0)) +} + +/// Rejects the push when `concurrency_key` already has `CONCURRENCY_KEY_MAX_QUEUED` jobs queued. +/// +/// Caller must runtime-gate this on `*CLOUD_HOSTED`, and must call it from *every push path* +/// that registers a concurrency key: a flow with a preprocessor is pushed with `concurrent_limit` +/// cleared and only registers its key later from `worker_flow`, so gating `push` alone leaves +/// trigger-driven flows uncapped. +/// +/// `add_batch_jobs` writes keys directly and is deliberately exempt: it is superadmin-only, and a +/// superadmin can set the cap to `0` anyway. `import_queued_jobs` is likewise exempt because it +/// is rejected outright on cloud. +#[cfg(feature = "cloud")] +async fn check_concurrency_key_queue_cap<'c>( + db: impl PgExecutor<'c>, + concurrency_key: &str, +) -> Result<(), Error> { + let cap = windmill_common::worker::CONCURRENCY_KEY_MAX_QUEUED + .load(std::sync::atomic::Ordering::Relaxed); + if cap == 0 { + return Ok(()); + } + let cap = cap as i64; + let depth = concurrency_key_queue_depth(db, concurrency_key, cap).await?; + if depth >= cap { + return Err(Error::QuotaExceeded(format!( + "Too many jobs queued behind concurrency key '{concurrency_key}': at least {depth} \ + jobs are already waiting and the limit is {cap}. Jobs sharing a concurrency key run \ + at most `concurrent_limit` at a time, so this queue is growing faster than it can \ + drain. Cancel the backlog, slow down the caller, or raise the concurrency limit." + ))); + } + Ok(()) +} + +/// Bounded count of a workspace's non-running queued jobs, capped at `limit` so the scan +/// stops once the ceiling is reached rather than counting an entire runaway backlog. +/// +/// Internal helper for `check_workspace_queue_cap`, `pub` only so the integration test can call +/// it (like `concurrency_key_queue_depth`). Returns a count, not job contents; the caller is +/// responsible for any authorization — it takes the workspace id as given. +pub async fn workspace_queue_depth<'c>( + db: impl PgExecutor<'c>, + workspace_id: &str, + limit: i64, +) -> Result { + sqlx::query_scalar!( + "SELECT count(*) FROM ( + SELECT 1 FROM v2_job_queue + WHERE workspace_id = $1 AND running = false + LIMIT $2 + ) s", + workspace_id, + limit, + ) + .fetch_one(db) + .warn_after_seconds(3) + .await + .map_err(|e| { + Error::internal_err(format!( + "Could not count queued jobs for workspace={workspace_id}: {e:#}" + )) + }) + .map(|c| c.unwrap_or(0)) +} + +/// Rejects the push when the workspace already has `WORKSPACE_MAX_QUEUED_JOBS` jobs queued. +/// +/// Caller must runtime-gate this on `*CLOUD_HOSTED`. It runs on every push (not only +/// concurrency-limited ones) because a workspace can flood the queue across many keys or with +/// keyless jobs. It caps *new* pushes past the ceiling; jobs already queued still drain, so an +/// in-flight flow only ever fails to push further work while the workspace is at the ceiling. +/// +/// This is a soft ceiling, like the per-key cap: the count and the insert are not serialized, so +/// a burst of concurrent pushes can land a handful over the limit. That is fine and intentional +/// — the cap exists to stop an unbounded runaway, not to enforce an exact quota, and a +/// per-workspace lock on every push would add hot-path contention for no practical gain. +#[cfg(feature = "cloud")] +async fn check_workspace_queue_cap<'c>( + db: impl PgExecutor<'c>, + workspace_id: &str, +) -> Result<(), Error> { + let cap = windmill_common::worker::WORKSPACE_MAX_QUEUED_JOBS + .load(std::sync::atomic::Ordering::Relaxed); + if cap == 0 { + return Ok(()); + } + let cap = cap as i64; + let depth = workspace_queue_depth(db, workspace_id, cap).await?; + if depth >= cap { + return Err(Error::QuotaExceeded(format!( + "Too many jobs queued in workspace '{workspace_id}': at least {depth} jobs are \ + already waiting and the instance limit is {cap}. Cancel the backlog or slow down \ + whatever is creating jobs before pushing more." + ))); + } + Ok(()) +} + // pub async fn insert_debounce_key<'d, 'c>( // workspace_id: &str, // args: &PushArgs<'d>, @@ -6812,6 +7135,78 @@ fn create_restarted_module( } } +/// A between-steps-zombie step: an `InProgress` module (in an otherwise terminal, +/// reaped flow) whose every child is recorded as a `success` completion. Only the +/// module's final state transition was lost, so the whole step is derivable and +/// safe to reuse on restart. Children incomplete/failed/cancelled ⟹ not a zombie. +async fn is_derivable_between_steps_zombie( + db: &Pool, + workspace_id: &str, + module: &FlowStatusModule, +) -> Result { + // The module's own cursor must prove it reached the end (a serial loop/branch-all reaped + // mid-fan-out has an all-success prefix but unrun remaining iterations); while-loops are + // never derivable. Children-success is verified below. + if !module.is_between_steps_complete() { + return Ok(false); + } + let child_ids: Vec = module + .flow_jobs() + .filter(|v| !v.is_empty()) + .or_else(|| module.job().map(|j| vec![j])) + .unwrap_or_default(); + if child_ids.is_empty() { + return Ok(false); + } + let success_children = sqlx::query_scalar!( + "SELECT count(*) FROM v2_job_completed + WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'", + workspace_id, + &child_ids, + ) + .fetch_one(db) + .await? + .unwrap_or(0); + Ok(success_children == child_ids.len() as i64) +} + +/// Convert a between-steps-zombie `InProgress` module (validated by +/// [`is_derivable_between_steps_zombie`]) into the `Success` it would have become +/// had its dropped transition landed, reusing all completed children. Downstream +/// steps re-derive this step's result from `flow_jobs`/`job` on demand +/// (`get_previous_job_result`), so no aggregate needs recomputing here. +fn reuse_completed_zombie_module(module: FlowStatusModule) -> FlowStatusModule { + match module { + FlowStatusModule::InProgress { + id, + job, + flow_jobs, + flow_jobs_success, + flow_jobs_duration, + branch_chosen, + agent_actions, + agent_actions_success, + .. + } => FlowStatusModule::Success { + id, + job, + // Every child was verified successful, so normalise the success + // vector (the dropped transition may have left the last entry unset). + flow_jobs_success: flow_jobs_success + .map(|v| v.into_iter().map(|_| Some(true)).collect()), + flow_jobs, + flow_jobs_duration, + branch_chosen, + approvers: vec![], + failed_retries: vec![], + skipped: false, + agent_actions, + agent_actions_success, + }, + other => other, + } +} + async fn restarted_flows_resolution( db: &Pool, workspace_id: &str, @@ -6819,6 +7214,15 @@ async fn restarted_flows_resolution( restart_step_id: &str, branch_or_iteration_n: Option, flow_version: Option, + // A nested restart chain (RestartedFrom.nested) descends into the restart step's child to + // re-run an inner step; zombie reuse would skip the whole container and ignore it. + nested_restart: bool, + // Zombie reuse validates the restart step against the completed job's STORED definition and + // synthesizes Success from its recorded children. That is only sound when the run being queued + // uses that same definition (JobPayload::RestartedFlow). A JobPayload::RawFlow restart queues + // the editor's current, possibly EDITED, definition instead, so reuse would skip the edited + // step and reuse the old child result; disable it there. + allow_zombie_reuse: bool, ) -> Result< ( Option, @@ -6835,7 +7239,7 @@ async fn restarted_flows_resolution( let row = sqlx::query!( "SELECT j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\", - j.kind AS \"job_kind!: JobKind\", + j.kind AS \"job_kind!: JobKind\", c.canceled_by, COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json>\", j.raw_flow AS \"raw_flow: Json>\" FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2", @@ -6851,6 +7255,12 @@ async fn restarted_flows_resolution( )) })?; + // Zombie reuse must only apply to flows the zombie monitor reaped (canceled_by = 'monitor'). + // An ordinary force-cancel copies the same live flow_status, so a user canceling after a child + // succeeds but before the parent transition lands produces the identical InProgress/all-success + // shape; those must retain restart-from-step semantics (the step re-runs). + let reaped_by_monitor = row.canceled_by.as_deref() == Some("monitor"); + let current_flow_version = row.script_hash.map(|x| x.0); let is_version_change = flow_version.is_some() && current_flow_version.is_some() @@ -6941,9 +7351,36 @@ async fn restarted_flows_resolution( continue; }; if module.id() == restart_step_id { - // if the module ID is the one we want to restart the flow at, or if it's past it in the flow, - // set the module as WaitingForPriorSteps as it needs to be re-run - if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 { + // Reuse is only safe when there is a NEXT step to advance into (advancing past the + // last module lands on the failure step) and the step's definition carries no + // completion/arming semantics that reuse would skip (stop predicates, skip_if, + // suspend, sleep); such a step must re-run, not be synthesized as Success. + let has_next_step = flow_value + .modules + .last() + .is_none_or(|m| m.id != restart_step_id); + // A whole-step restart is `None` (restart API with the field omitted) or `Some(0)` + // (the run page's "Re-start from" button always sends 0); both mean "redo this step", + // which for a monitor-reaped zombie means reuse it. `Some(n>=1)` is an explicit + // partial container restart and keeps its existing reuse-0..n-1 / rerun-from-n path. + if allow_zombie_reuse + && reaped_by_monitor + && branch_or_iteration_n.unwrap_or(0) == 0 + && !nested_restart + && has_next_step + && module_definition.allows_zombie_reuse() + && is_derivable_between_steps_zombie(db, workspace_id, &module).await? + { + // Between-steps-zombie recovery: this step's children all + // completed but its final state transition was dropped (the + // flow was reaped by the zombie monitor). Reuse the completed + // step verbatim and restart from the NEXT step, so no child + // re-runs and only the dropped transition is replayed onward. + step_n += 1; + truncated_modules.push(reuse_completed_zombie_module(module)); + } else if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 { + // if the module ID is the one we want to restart the flow at, or if it's past it in the flow, + // set the module as WaitingForPriorSteps as it needs to be re-run // The module as WaitingForPriorSteps as the entire module (i.e. all the branches) need to be re-run truncated_modules .push(FlowStatusModule::WaitingForPriorSteps { id: module.id() }); diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 2b5aef96a5..55944b08ca 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -597,6 +597,121 @@ pub async fn push_scheduled_job<'c>( Ok(tx) // TODO: Bubble up pushed UUID from here } +/// Enabled schedules with no occurrence in the queue, as `(workspace_id, path)`. +/// +/// Every path that completes a scheduled job pushes the next occurrence in the +/// same transaction (for flows, on entry to step 0), so an enabled schedule +/// always has a queued occurrence — a run in progress is itself one. A run that +/// dies through an abnormal path can skip that push though, leaving the schedule +/// enabled yet dead until it is manually disabled and re-enabled. This is how the +/// monitor spots that state; see `rearm_schedule` for the recovery. +/// +/// Not an authorization boundary: it reports schedules across every workspace, so +/// this is for system callers (the monitor's reconciliation pass) only and its +/// result must never be returned to a user unfiltered. +pub async fn find_unarmed_schedules(db: &DB) -> Result> { + let rows = sqlx::query!( + // Query plan: the anti-join builds from `v2_job_queue` (only pending and + // running jobs) rather than probing `v2_job` once per schedule. + "SELECT s.workspace_id, s.path + FROM schedule s JOIN workspace w ON w.id = s.workspace_id AND NOT w.deleted + WHERE s.enabled IS TRUE + AND NOT EXISTS ( + SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id) + WHERE j.workspace_id = s.workspace_id + AND j.trigger_kind = 'schedule' + AND j.trigger = s.path + AND j.runnable_path = s.script_path + AND j.parent_job IS NULL + )" + ) + .fetch_all(db) + .await?; + Ok(rows.into_iter().map(|r| (r.workspace_id, r.path)).collect()) +} + +#[derive(Debug, PartialEq, Eq)] +pub enum RearmOutcome { + /// The next occurrence was pushed. + Rearmed, + /// Nothing to do: the schedule was deleted or disabled since it was found. + NoOp, +} + +/// Push the next occurrence of a schedule that has none queued. +/// +/// Only ever starts a schedule, never stops one: re-arming something that did not +/// need it costs one extra run, whereas wrongly disabling one is the silent +/// permanent stoppage this whole mechanism exists to prevent. So an occurrence +/// that cannot be pushed is logged and left alone — the schedule is already not +/// running, and `try_schedule_next_job` still disables on the completion path, +/// where the population is limited to actively-cycling schedules. Keep it that +/// way: this sweeps *every* enabled schedule, including ones broken long before +/// this code existed and never swept before. +/// +/// Not an authorization boundary: it pushes under the schedule's own +/// `permissioned_as` identity for any `(w_id, path)`, so this is for system +/// callers (the monitor's reconciliation pass) only. A caller acting for a user +/// MUST already have enforced their permissions on `w_id` and `path`. +pub async fn rearm_schedule(db: &DB, w_id: &str, path: &str) -> Result { + let mut tx = db.begin().await?; + // Lock the row for the whole push: an edit or a disable committing between the + // read and the push would otherwise leave a queued occurrence for a schedule + // that is disabled, or one built from superseded settings. + let schedule = sqlx::query_as::<_, Schedule>( + "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule WHERE path = $1 AND workspace_id = $2 FOR UPDATE", + ) + .bind(path) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + let Some(schedule) = schedule else { + return Ok(RearmOutcome::NoOp); + }; + if !schedule.enabled { + return Ok(RearmOutcome::NoOp); + } + // Re-check for a queued occurrence now that the row is locked: a normal + // completion, an edit, or a re-enable could have pushed one between the unarmed + // scan and this lock. push_scheduled_job only dedups the exact computed + // scheduled_for, so re-arming a schedule that has since become armed and crossed a + // cron boundary would queue a second root occurrence. Mirrors the anti-join in + // find_unarmed_schedules. + let already_armed: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM v2_job_queue q JOIN v2_job j USING (id) + WHERE j.workspace_id = $1 + AND j.trigger_kind = 'schedule' + AND j.trigger = $2 + AND j.runnable_path = $3 + AND j.parent_job IS NULL + )", + ) + .bind(w_id) + .bind(path) + .bind(&schedule.script_path) + .fetch_one(&mut *tx) + .await?; + if already_armed { + return Ok(RearmOutcome::NoOp); + } + match push_scheduled_job(db, tx, &schedule, None, None).await { + Ok(tx) => { + tx.commit().await?; + Ok(RearmOutcome::Rearmed) + } + // An occurrence that can never be pushed (runnable gone, quota blown) is + // reported, not acted on — see the note above on why this never disables. + Err(err @ (error::Error::NotFound(_) | error::Error::QuotaExceeded(_))) => { + tracing::error!( + "Could not re-arm schedule {path} in {w_id}: {err}. Leaving it enabled; it will not run until the cause is fixed." + ); + Ok(RearmOutcome::NoOp) + } + Err(err) => Err(err), + } +} + pub async fn get_schedule_opt<'c>( e: impl PgExecutor<'c>, w_id: &str, diff --git a/backend/windmill-queue/tests/concurrency_key_queue_depth_test.rs b/backend/windmill-queue/tests/concurrency_key_queue_depth_test.rs new file mode 100644 index 0000000000..3689289bf4 --- /dev/null +++ b/backend/windmill-queue/tests/concurrency_key_queue_depth_test.rs @@ -0,0 +1,105 @@ +//! Regression guard for `concurrency_key_queue_depth`, which backs the cloud-only cap on how +//! many jobs may queue behind a single concurrency key. +//! +//! Run with: +//! cargo test -p windmill-queue --test concurrency_key_queue_depth_test + +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::jobs::concurrency_key_queue_depth; + +/// Queues `count` jobs on `key`, all scheduled `offset_secs` from now. +async fn seed_queued(db: &Pool, key: &str, count: usize, offset_secs: i64) { + seed(db, key, count, offset_secs, false).await +} + +async fn seed(db: &Pool, key: &str, count: usize, offset_secs: i64, running: bool) { + for _ in 0..count { + let id = Uuid::new_v4(); + sqlx::query!( + "INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, 'test-workspace', 'other')", + id, + ) + .execute(db) + .await + .expect("seed v2_job"); + sqlx::query!( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running) + VALUES ($1, 'test-workspace', now() + ($2::bigint::text || ' s')::interval, 'other', $3)", + id, + offset_secs, + running, + ) + .execute(db) + .await + .expect("seed v2_job_queue"); + sqlx::query!( + "INSERT INTO concurrency_key (key, job_id) VALUES ($1, $2)", + key, + id, + ) + .execute(db) + .await + .expect("seed concurrency_key"); + } +} + +/// The property the cap depends on: jobs scheduled into the future count toward the depth. +/// +/// The concurrency limiter re-queues a blocked job by pushing `scheduled_for` forward one full +/// window at a time, so a backlog gated by a concurrency limit is almost entirely future-dated. +/// A depth query narrowed to `scheduled_for <= now()` would report only the trickle the limiter +/// has released and the cap would never fire on the runaway it exists to stop. +#[sqlx::test(migrations = "../migrations")] +async fn counts_future_scheduled_jobs(db: Pool) { + let key = "future-scheduled-count"; + seed_queued(&db, key, 3, -10).await; // due now + seed_queued(&db, key, 7, 3600).await; // parked an hour out by the limiter + + let depth = concurrency_key_queue_depth(&db, key, 1000) + .await + .expect("count depth"); + + assert_eq!( + depth, 10, + "depth must include future-scheduled jobs; counting only due jobs would report 3" + ); +} + +/// Running jobs are not backlog. They stay in `v2_job_queue` with `ended_at` still NULL, so +/// counting them would charge a key for the concurrency it is licensed to use: a key whose +/// `concurrent_limit` exceeds the cap would reject every push with nothing actually waiting. +#[sqlx::test(migrations = "../migrations")] +async fn ignores_running_jobs(db: Pool) { + let key = "running-not-backlog"; + seed(&db, key, 6, -10, true).await; // executing right now + seed(&db, key, 2, 3600, false).await; // actually waiting + + let depth = concurrency_key_queue_depth(&db, key, 1000) + .await + .expect("count depth"); + + assert_eq!( + depth, 2, + "only waiting jobs count; the 6 running ones must not" + ); +} + +/// A job that left the queue without going through `add_completed_job` leaves its +/// `concurrency_key` row with `ended_at IS NULL` forever. Those must not count, otherwise a mass +/// cancel or manual purge leaves the key permanently wedged above the cap with an empty queue. +#[sqlx::test(migrations = "../migrations")] +async fn ignores_rows_whose_job_left_the_queue(db: Pool) { + let key = "orphaned-rows"; + seed_queued(&db, key, 4, 3600).await; + sqlx::query!("DELETE FROM v2_job_queue") + .execute(&db) + .await + .expect("purge queue"); + + let depth = concurrency_key_queue_depth(&db, key, 1000) + .await + .expect("count depth"); + + assert_eq!(depth, 0, "orphaned concurrency_key rows must not count"); +} diff --git a/backend/windmill-queue/tests/concurrency_limit_zero_test.rs b/backend/windmill-queue/tests/concurrency_limit_zero_test.rs new file mode 100644 index 0000000000..bd72571bcd --- /dev/null +++ b/backend/windmill-queue/tests/concurrency_limit_zero_test.rs @@ -0,0 +1,21 @@ +//! Runtime gate for the `Some(0)` concurrency footgun: a stored `concurrent_limit <= 0` +//! must read as "disabled", never as a zero-slot cap that permanently blocks the job at the +//! concurrency gate (the re-queue storm the zombie monitor eventually fails as a fake OOM). +//! +//! Run with: +//! cargo test -p windmill-queue --test concurrency_limit_zero_test + +use windmill_queue::jobs::has_active_concurrency_limit; + +#[test] +fn zero_and_negative_are_not_active_limits() { + assert!(!has_active_concurrency_limit(None)); + assert!(!has_active_concurrency_limit(Some(0))); + assert!(!has_active_concurrency_limit(Some(-1))); +} + +#[test] +fn positive_limit_is_active() { + assert!(has_active_concurrency_limit(Some(1))); + assert!(has_active_concurrency_limit(Some(i32::MAX))); +} diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index e11ced23bc..3ac4cd2bd9 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -10,7 +10,9 @@ mod schedule_push { use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; use windmill_queue::jobs::{try_schedule_next_job, MiniCompletedJob}; - use windmill_queue::schedule::push_scheduled_job; + use windmill_queue::schedule::{ + find_unarmed_schedules, push_scheduled_job, rearm_schedule, RearmOutcome, + }; fn make_schedule(overrides: impl FnOnce(&mut Schedule)) -> Schedule { let mut s = Schedule { @@ -1427,172 +1429,6 @@ mod schedule_push { } } - // =================================================================== - // Zombie detection tests — verify that a flow left in queue after - // SchedulePushZombieError meets the restart criteria in monitor.rs - // =================================================================== - - // ----------------------------------------------------------------------- - // When both schedule push AND post-retry disable fail, the flow job stays - // in v2_job_queue as a zombie. This test simulates that state and verifies: - // - // 1. The zombie detection query (from handle_zombie_flows) finds the flow - // 2. The flow meets restart criteria: first module is WaitingForPriorSteps - // and same_worker is false - // 3. After the restart UPDATE, the flow is re-queued (running=false) - // 4. The schedule remains enabled for retry on next flow execution - // ----------------------------------------------------------------------- - - #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] - async fn test_zombie_flow_after_schedule_push_failure_meets_restart_criteria( - db: Pool, - ) -> anyhow::Result<()> { - use windmill_common::flow_status::{FlowStatus, FlowStatusModule}; - - let flow_job_id = uuid::Uuid::new_v4(); - let now = Utc::now(); - let stale_ping = now - chrono::Duration::minutes(5); - - // Schedule still enabled — simulates both push and disable failing - sqlx::query( - "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap, permissioned_as) - VALUES ('test-workspace', 'f/system/test_schedule', 'test-user', now(), '0 0 */5 * * *', 'UTC', true, 'f/system/test_flow', true, 'test@windmill.dev', '{}', false, false, 'u/test-user')" - ) - .execute(&db) - .await?; - - // Flow job in v2_job (kind=flow, triggered by schedule, same_worker=false) - sqlx::query( - "INSERT INTO v2_job (id, workspace_id, created_at, created_by, permissioned_as, permissioned_as_email, kind, runnable_path, trigger, trigger_kind, same_worker, visible_to_owner, tag) - VALUES ($1, 'test-workspace', $2, 'test-user', 'u/test-user', 'test@windmill.dev', 'flow', 'f/system/test_flow', 'f/system/test_schedule', 'schedule', false, false, 'flow')" - ) - .bind(flow_job_id) - .bind(now - chrono::Duration::minutes(5)) - .execute(&db) - .await?; - - // Queue entry: running=true (worker caught SchedulePushZombieError and returned Ok) - sqlx::query( - "INSERT INTO v2_job_queue (id, workspace_id, created_at, scheduled_for, running, started_at, tag, suspend) - VALUES ($1, 'test-workspace', $2, $2, true, $3, 'flow', 0)" - ) - .bind(flow_job_id) - .bind(now - chrono::Duration::minutes(5)) - .bind(now - chrono::Duration::minutes(4)) - .execute(&db) - .await?; - - // Stale ping — older than the 60s zombie transition timeout - sqlx::query("INSERT INTO v2_job_runtime (id, ping) VALUES ($1, $2)") - .bind(flow_job_id) - .bind(stale_ping) - .execute(&db) - .await?; - - // Flow status at step 0, first module = WaitingForPriorSteps - // This is the initial state of a flow that hasn't started any steps yet - let flow_status = serde_json::json!({ - "step": 0, - "modules": [{"type": "WaitingForPriorSteps", "id": "a"}], - "failure_module": {"type": "WaitingForPriorSteps", "id": "failure"}, - "retry": {"fail_count": 0, "failed_jobs": []}, - "cleanup_module": {"flow_jobs_to_clean": []} - }); - - sqlx::query("INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2::jsonb)") - .bind(flow_job_id) - .bind(&flow_status) - .execute(&db) - .await?; - - // Run the same zombie detection query from handle_zombie_flows (60s timeout) - let zombie_flows = sqlx::query_as::<_, (uuid::Uuid, String, Option, Option)>( - r#" - SELECT - j.id, j.workspace_id, j.same_worker, - COALESCE(s.flow_status, s.workflow_as_code_status)::text AS flow_status - FROM v2_job_queue q - JOIN v2_job j USING (id) - LEFT JOIN v2_job_runtime r USING (id) - LEFT JOIN v2_job_status s USING (id) - WHERE q.running = true AND q.suspend = 0 AND q.suspend_until IS null - AND q.scheduled_for <= now() - AND (j.kind = 'flow' OR j.kind = 'flowpreview' OR j.kind = 'flownode' OR j.kind = 'singlestepflow') - AND r.ping IS NOT NULL - AND r.ping < NOW() - ('60' || ' seconds')::interval - AND q.canceled_by IS NULL - "#, - ) - .fetch_all(&db) - .await?; - - assert_eq!(zombie_flows.len(), 1, "zombie flow must be detected"); - let (id, _ws, same_worker, flow_status_json) = &zombie_flows[0]; - assert_eq!(*id, flow_job_id); - - // Replicate the exact branching logic from handle_zombie_flows (monitor.rs:2711-2754). - // Only flows matching the restart condition get restarted; others are cancelled. - let status = flow_status_json - .as_deref() - .and_then(|x| serde_json::from_str::(x).ok()); - let should_restart = !same_worker.unwrap_or(false) - && status.is_some_and(|s| { - s.modules - .get(0) - .is_some_and(|x| matches!(x, FlowStatusModule::WaitingForPriorSteps { .. })) - }); - - assert!( - should_restart, - "flow must match the restart branch (not the cancel branch) in handle_zombie_flows" - ); - - // Apply the restart action — same UPDATE as handle_zombie_flows - sqlx::query( - "UPDATE v2_job_queue SET running = false, started_at = null - WHERE id = $1 AND canceled_by IS NULL", - ) - .bind(flow_job_id) - .execute(&db) - .await?; - - // Flow is re-queued for processing - let (running, started_at): (bool, Option>) = - sqlx::query_as("SELECT running, started_at FROM v2_job_queue WHERE id = $1") - .bind(flow_job_id) - .fetch_one(&db) - .await?; - assert!(!running, "flow must not be running after zombie restart"); - assert!( - started_at.is_none(), - "started_at must be null after zombie restart" - ); - - // Schedule still enabled — will be retried when flow re-executes - let enabled: bool = sqlx::query_scalar( - "SELECT enabled FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/test_schedule'", - ) - .fetch_one(&db) - .await?; - assert!( - enabled, - "schedule must remain enabled for retry after zombie restart" - ); - - // Flow job is NOT in v2_job_completed (it was never completed with error) - let completed_count: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM v2_job_completed WHERE id = $1") - .bind(flow_job_id) - .fetch_one(&db) - .await?; - assert_eq!( - completed_count, 0, - "flow must not be in completed_job — it's a zombie, not an error" - ); - - Ok(()) - } - // ----------------------------------------------------------------------- // push_scheduled_job: reserved ducklake-maintenance prefix // ----------------------------------------------------------------------- @@ -1762,4 +1598,121 @@ mod schedule_push { assert!(!row_exists, "managed schedule row must be deleted"); Ok(()) } + + // ----------------------------------------------------------------------- + // find_unarmed_schedules / rearm_schedule: recovery for a schedule left + // enabled with no queued occurrence (a run that died on an abnormal path + // skipped its next-occurrence push). Without this the chain stays dead + // until the schedule is manually disabled and re-enabled. + // ----------------------------------------------------------------------- + + async fn insert_schedule(db: &Pool, path: &str, script_path: &str, enabled: bool) { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, email, extra_perms, ws_error_handler_muted, no_flow_overlap, permissioned_as) + VALUES ('test-workspace', $1, 'test-user', now(), '0 0 */5 * * *', 'UTC', $3, $2, false, 'test@windmill.dev', '{}', false, true, 'u/test-user')", + ) + .bind(path) + .bind(script_path) + .bind(enabled) + .execute(db) + .await + .unwrap(); + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_find_unarmed_schedules(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + insert_schedule(&db, "f/system/disabled", "f/system/test_script", false).await; + + // No occurrence queued yet: the enabled schedule is unarmed, the disabled one is ignored. + assert_eq!( + find_unarmed_schedules(&db).await?, + vec![( + "test-workspace".to_string(), + "f/system/test_schedule".to_string() + )] + ); + + // Once an occurrence is queued it is armed and must not be reported — + // re-arming it would double-push the occurrence. + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &make_schedule(|_| {}), None, None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + assert!(find_unarmed_schedules(&db).await?.is_empty()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_pushes_next_occurrence(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::Rearmed + ); + + assert_eq!(count_queued_jobs(&db).await, 1); + assert!(find_unarmed_schedules(&db).await?.is_empty()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_skips_disabled(db: Pool) -> anyhow::Result<()> { + // A disable that lands between the scan and the re-arm must win: pushing an + // occurrence for a disabled schedule would resurrect a schedule the user + // just turned off. + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", false).await; + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::NoOp + ); + assert_eq!(count_queued_jobs(&db).await, 0); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_never_disables(db: Pool) -> anyhow::Result<()> { + insert_schedule(&db, "f/system/bad_schedule", "f/system/nonexistent", true).await; + + // Reconciliation only ever starts a schedule. An unpushable occurrence is + // reported and left alone: this sweeps every enabled schedule in the + // instance, so disabling here would turn a wrong invariant into the exact + // silent stoppage the reconciler exists to undo. + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/bad_schedule").await?, + RearmOutcome::NoOp + ); + + assert_eq!(count_queued_jobs(&db).await, 0); + let (enabled, error): (bool, Option) = sqlx::query_as( + "SELECT enabled, error FROM schedule WHERE workspace_id = 'test-workspace' AND path = 'f/system/bad_schedule'", + ) + .fetch_one(&db) + .await?; + assert!(enabled, "reconciliation must never disable a schedule"); + assert!(error.is_none()); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_rearm_schedule_skips_already_armed(db: Pool) -> anyhow::Result<()> { + // An occurrence can be queued (a normal completion, an edit, a re-enable) + // between the unarmed scan and rearm_schedule acquiring the row lock. Re-arming + // then would double-push, since push_scheduled_job only dedups the exact + // computed scheduled_for. + insert_schedule(&db, "f/system/test_schedule", "f/system/test_script", true).await; + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &make_schedule(|_| {}), None, None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + + assert_eq!( + rearm_schedule(&db, "test-workspace", "f/system/test_schedule").await?, + RearmOutcome::NoOp + ); + assert_eq!(count_queued_jobs(&db).await, 1); + Ok(()) + } } diff --git a/backend/windmill-queue/tests/workspace_queue_depth_test.rs b/backend/windmill-queue/tests/workspace_queue_depth_test.rs new file mode 100644 index 0000000000..f6969afc4c --- /dev/null +++ b/backend/windmill-queue/tests/workspace_queue_depth_test.rs @@ -0,0 +1,71 @@ +//! Regression guard for `workspace_queue_depth`, which backs the cloud-only cap on how many +//! jobs a single workspace may have queued in total. +//! +//! Run with: +//! cargo test -p windmill-queue --test workspace_queue_depth_test + +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::jobs::workspace_queue_depth; + +/// Queues `count` jobs in `workspace`, scheduled `offset_secs` from now, with `running` state. +async fn seed(db: &Pool, workspace: &str, count: usize, offset_secs: i64, running: bool) { + for _ in 0..count { + let id = Uuid::new_v4(); + sqlx::query!( + "INSERT INTO v2_job (id, workspace_id, tag) VALUES ($1, $2, 'other')", + id, + workspace, + ) + .execute(db) + .await + .expect("seed v2_job"); + sqlx::query!( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running) + VALUES ($1, $2, now() + ($3::bigint::text || ' s')::interval, 'other', $4)", + id, + workspace, + offset_secs, + running, + ) + .execute(db) + .await + .expect("seed v2_job_queue"); + } +} + +/// The count is per workspace and ignores running jobs: a runaway in one workspace must not +/// charge another, and jobs already executing are not backlog. Future-scheduled jobs count, +/// since a concurrency-parked backlog is almost entirely future-dated. +#[sqlx::test(migrations = "../migrations")] +async fn scoped_to_workspace_and_ignores_running(db: Pool) { + seed(&db, "ws-a", 4, 3600, false).await; // waiting (parked in the future) + seed(&db, "ws-a", 3, -10, false).await; // waiting (due now) + seed(&db, "ws-a", 5, -10, true).await; // running — not backlog + seed(&db, "ws-b", 9, 3600, false).await; // a different workspace + + let depth = workspace_queue_depth(&db, "ws-a", 1000) + .await + .expect("count depth"); + + assert_eq!( + depth, 7, + "only ws-a's 7 waiting jobs count; running jobs and ws-b must not" + ); +} + +/// The scan stops at `limit`, so a runaway backlog does not cost an unbounded count on every +/// push. The cap only needs to know the depth has reached the ceiling. +#[sqlx::test(migrations = "../migrations")] +async fn bounded_by_limit(db: Pool) { + seed(&db, "ws-a", 50, -10, false).await; + + let depth = workspace_queue_depth(&db, "ws-a", 10) + .await + .expect("count depth"); + + assert_eq!( + depth, 10, + "the count must stop at the limit, not scan all 50" + ); +} diff --git a/backend/windmill-store/Cargo.toml b/backend/windmill-store/Cargo.toml index fdd82a7e5d..1409e30ebf 100644 --- a/backend/windmill-store/Cargo.toml +++ b/backend/windmill-store/Cargo.toml @@ -19,6 +19,7 @@ mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] http_trigger = [] postgres_trigger = [] mqtt_trigger = [] +amqp_trigger = [] sqs_trigger = [] gcp_trigger = [] azure_trigger = [] diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index c70314430d..4ccf364cf8 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -184,6 +184,7 @@ struct EditResource { path: Option, description: Option, value: Option>, + resource_type: Option, labels: Option>, ws_specific: Option, } @@ -242,12 +243,8 @@ async fn list_search_resources( Extension(user_db): Extension, ) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] let n = 1000; - #[cfg(not(feature = "enterprise"))] - let n = 3; - let allowed = build_scope_path_predicate(&authed, "resources", "read"); let rows = sqlx::query_as!( SearchResource, @@ -1717,6 +1714,9 @@ async fn update_resource( if let Some(nvalue) = &ns.value { sqlb.set_str("value", nvalue.to_string()); } + if let Some(nrt) = &ns.resource_type { + sqlb.set_str("resource_type", nrt); + } if let Some(ndesc) = ns.description { sqlb.set_str("description", ndesc); } @@ -2386,6 +2386,7 @@ async fn update_resource_type( feature = "http_trigger", feature = "postgres_trigger", feature = "mqtt_trigger", + feature = "amqp_trigger", all( feature = "enterprise", any( @@ -2568,6 +2569,13 @@ async fn validate_git_url(url: &str) -> Result<()> { let host = extract_host_from_git_url(url) .ok_or_else(|| Error::BadRequest("Could not parse hostname from git URL".to_string()))?; + // CI/dev escape hatch: integration tests run their git remote (a Gitea + // container) on localhost, which the network-target checks below reject. + // Scheme and option-injection validation above still applies. + if std::env::var("ALLOW_LOCAL_GIT_REMOTES").is_ok_and(|v| v == "true" || v == "1") { + return Ok(()); + } + if host == "localhost" || host.ends_with(".local") || host == "[::1]" { return Err(Error::BadRequest( "Git URLs targeting localhost or local network are not allowed".to_string(), @@ -2781,6 +2789,26 @@ async fn get_git_ssh_cmd( Ok((Some(git_ssh_cmd), file_paths)) } +/// Run a git remote probe with a hard per-command deadline. The auto-pull poller +/// walks every repository sequentially in one monitor pass, so a single +/// unresponsive remote must not stall the whole pass (or leave a hung child +/// process behind — `kill_on_drop` reaps it when the timeout fires). +const GIT_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +async fn run_git_probe(mut git_cmd: Command, what: &str) -> Result { + git_cmd.kill_on_drop(true); + match tokio::time::timeout(GIT_PROBE_TIMEOUT, git_cmd.output()).await { + Ok(output) => { + output.map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e))) + } + Err(_) => Err(Error::internal_err(format!( + "git {} timed out after {}s", + what, + GIT_PROBE_TIMEOUT.as_secs() + ))), + } +} + async fn get_repo_latest_commit_hash( git_resource: &GitRepositoryResource, git_ssh_command: Option, @@ -2806,10 +2834,7 @@ async fn get_repo_latest_commit_hash( } git_cmd.stderr(Stdio::piped()); - let output = git_cmd - .output() - .await - .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; + let output = run_git_probe(git_cmd, "ls-remote").await?; if !output.status.success() { let stderr = String::from_utf8(output.stderr) @@ -2843,6 +2868,234 @@ async fn get_repo_latest_commit_hash( Ok(commit_hash) } +/// Load a git-sync repository resource's value with `$var:`/`$res:` references +/// resolved. Shared by the auto-pull poller (`get_git_repo_head_for_autopull`) +/// and deploy-mode detection so the interpolation lives in exactly one place. +/// +/// SECURITY: reads under the system identity (`SUPERADMIN_SYNC_EMAIL`), so it +/// **bypasses resource RLS** and returns fully-interpolated JSON that **may +/// contain credentials** (an embedded `$var:` token in the URL). Callers must +/// have already authorized access to `w_id`, must use it only for git-sync +/// `git_repository` resources, and must **not** return the resolved value to a +/// client — derive and return only non-sensitive facts. Pass `allow_cache=true` +/// for the poller (avoids re-decrypting/re-auditing a `$var:` secret every tick); +/// pass `false` for on-demand reads that must reflect the current resource. +pub async fn resolve_git_repository_resource( + db: &DB, + w_id: &str, + git_repo_resource_path: &str, + allow_cache: bool, +) -> Result> { + use windmill_common::db::DbWithOptAuthed; + + let resource_path = git_repo_resource_path + .strip_prefix("$res:") + .unwrap_or(git_repo_resource_path); + + let dba: DbWithOptAuthed<'_, ApiAuthed> = DbWithOptAuthed::DB { + db: db.clone(), + audit_author: windmill_common::audit::AuditAuthor { + username: "git_sync_auto_pull".to_string(), + email: windmill_common::users::SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + }, + }; + + get_resource_value_interpolated_internal(&dba, w_id, resource_path, None, None, allow_cache) + .await +} + +/// Resolve a workspace git-sync repository and return its current head commit +/// `(ref_spec, sha)` for the tracked branch, for background auto-pull polling. +/// Returns `Ok(None)` for repos that cannot be polled in-process (GitHub-App +/// repos, which sync via webhooks instead). +pub async fn get_git_repo_head_for_autopull( + db: &DB, + w_id: &str, + git_repo_resource_path: &str, +) -> Result> { + let value = resolve_git_repository_resource(db, w_id, git_repo_resource_path, true) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Git repository resource '{}' not found", + git_repo_resource_path + .strip_prefix("$res:") + .unwrap_or(git_repo_resource_path) + )) + })?; + + if value + .get("is_github_app") + .and_then(|v| v.as_bool()) + .unwrap_or(false) + { + return Ok(None); + } + + let git_resource: GitRepositoryResource = serde_json::from_value(value) + .map_err(|e| Error::BadRequest(format!("Invalid git repository resource: {}", e)))?; + + // The SSH identity is supplied per-call in the authed commit-hash path; the + // background poller has none, so an SSH remote can't authenticate here. Fail + // with an actionable message instead of a confusing ls-remote auth error — + // these repos should use an HTTPS token URL or the GitHub App for auto-pull. + let url = git_resource.url.trim_start(); + if !url.starts_with("http://") && !url.starts_with("https://") { + return Err(Error::BadRequest( + "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), + )); + } + + if let Some(branch) = git_resource.branch.as_deref().filter(|s| !s.is_empty()) { + let branch = branch.to_string(); + let sha = get_repo_latest_commit_hash(&git_resource, None).await?; + return Ok(Some((branch, sha))); + } + + // No explicit branch: resolve the remote's default-branch NAME along with + // its head in one call. Fork sync needs the concrete name to scope + // `wm-fork//*`, so a bare "HEAD" ref would silently disable it. + validate_git_url(&git_resource.url).await?; + let mut git_cmd = Command::new("git"); + git_cmd.args(["ls-remote", "--symref", &git_resource.url, "HEAD"]); + git_cmd.stderr(Stdio::piped()); + let output = run_git_probe(git_cmd, "ls-remote --symref HEAD").await?; + if !output.status.success() { + let stderr = String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Failed to decode stderr".to_string()); + return Err(Error::BadRequest(format!( + "Error resolving git repo HEAD: {}", + stderr + ))); + } + let stdout = String::from_utf8(output.stdout) + .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; + let (branch, sha) = parse_ls_remote_symref_head(&stdout); + let sha = sha.ok_or_else(|| { + Error::BadRequest(format!( + "No HEAD found in repository '{}'", + git_resource.url + )) + })?; + Ok(Some((branch.unwrap_or_else(|| "HEAD".to_string()), sha))) +} + +/// Parse `git ls-remote --symref HEAD` output: the `ref:` line names the +/// default branch, the plain line carries its head sha. +fn parse_ls_remote_symref_head(stdout: &str) -> (Option, Option) { + let mut branch = None; + let mut sha = None; + for line in stdout.lines() { + let mut parts = line.split_whitespace(); + match (parts.next(), parts.next()) { + (Some("ref:"), Some(target)) => { + if let Some(name) = target.strip_prefix("refs/heads/") { + branch = Some(name.to_string()); + } + } + (Some(hash), Some("HEAD")) => { + sha = Some(hash.to_string()); + } + _ => {} + } + } + (branch, sha) +} + +/// List the head sha of every `wm-fork//*` branch — plus any +/// `extra_refs` (dev workspaces' environment-label branches, e.g. `dev`, +/// `staging`) — of a workspace git-sync repository in one `git ls-remote` call, +/// for parent-managed fork sync polling. Same auth model and app-repo exclusion +/// as [`get_git_repo_head_for_autopull`]: returns `Ok(None)` for +/// GitHub-App-backed repos (polled over the API instead) and errors on SSH +/// remotes. +pub async fn get_git_repo_fork_heads_for_autopull( + db: &DB, + w_id: &str, + git_repo_resource_path: &str, + base_branch: &str, + extra_refs: &[String], +) -> Result>> { + use windmill_common::db::DbWithOptAuthed; + + let resource_path = git_repo_resource_path + .strip_prefix("$res:") + .unwrap_or(git_repo_resource_path); + + let dba: DbWithOptAuthed<'_, ApiAuthed> = DbWithOptAuthed::DB { + db: db.clone(), + audit_author: windmill_common::audit::AuditAuthor { + username: "git_sync_auto_pull".to_string(), + email: windmill_common::users::SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + }, + }; + let value = + get_resource_value_interpolated_internal(&dba, w_id, resource_path, None, None, true) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Git repository resource '{}' not found", + resource_path + )) + })?; + + if value + .get("is_github_app") + .and_then(|v| v.as_bool()) + .unwrap_or(false) + { + return Ok(None); + } + + let git_resource: GitRepositoryResource = serde_json::from_value(value) + .map_err(|e| Error::BadRequest(format!("Invalid git repository resource: {}", e)))?; + let url = git_resource.url.trim_start(); + if !url.starts_with("http://") && !url.starts_with("https://") { + return Err(Error::BadRequest( + "Automatic pull can't authenticate an SSH git remote in the background. Use an HTTPS URL with an embedded token, or connect the repository through the GitHub App.".to_string(), + )); + } + validate_git_url(&git_resource.url).await?; + validate_git_ref(base_branch)?; + + let mut git_cmd = Command::new("git"); + git_cmd.args([ + "ls-remote", + &git_resource.url, + &format!("refs/heads/wm-fork/{}/*", base_branch), + ]); + for r in extra_refs { + validate_git_ref(r)?; + git_cmd.arg(format!("refs/heads/{}", r)); + } + git_cmd.stderr(Stdio::piped()); + let output = run_git_probe(git_cmd, "ls-remote (fork branches)").await?; + if !output.status.success() { + let stderr = String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Failed to decode stderr".to_string()); + return Err(Error::BadRequest(format!( + "Error listing fork branches: {}", + stderr + ))); + } + let stdout = String::from_utf8(output.stdout) + .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; + let heads = stdout + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let sha = parts.next()?; + let branch = parts.next()?.strip_prefix("refs/heads/")?; + Some((branch.to_string(), sha.to_string())) + }) + .collect(); + Ok(Some(heads)) +} + #[cfg(all( feature = "enterprise", any(feature = "nats", feature = "kafka", feature = "sqs_trigger") @@ -2878,6 +3131,28 @@ mod tests { use windmill_common::audit::AuditAuthor; use windmill_common::db::DbWithOptAuthed; + #[test] + fn parse_symref_head_resolves_default_branch() { + let out = "ref: refs/heads/main\tHEAD\n7ddb8cec9a0000000000000000000000000000aa\tHEAD\n"; + assert_eq!( + parse_ls_remote_symref_head(out), + ( + Some("main".to_string()), + Some("7ddb8cec9a0000000000000000000000000000aa".to_string()) + ) + ); + // Detached/unknown symref: sha still parses, branch stays None. + let out2 = "1234567890000000000000000000000000000000\tHEAD\n"; + assert_eq!( + parse_ls_remote_symref_head(out2), + ( + None, + Some("1234567890000000000000000000000000000000".to_string()) + ) + ); + assert_eq!(parse_ls_remote_symref_head(""), (None, None)); + } + fn test_db_with_opt_authed(db: DB) -> DbWithOptAuthed<'static, ApiAuthed> { DbWithOptAuthed::DB { db, diff --git a/backend/windmill-trigger-amqp/Cargo.toml b/backend/windmill-trigger-amqp/Cargo.toml new file mode 100644 index 0000000000..4c5ce41941 --- /dev/null +++ b/backend/windmill-trigger-amqp/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "windmill-trigger-amqp" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_amqp" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["amqp_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +lapin.workspace = true +tokio-executor-trait.workspace = true +tokio-reactor-trait.workspace = true +futures.workspace = true +urlencoding.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +base64.workspace = true +anyhow.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-trigger-amqp/src/handler.rs b/backend/windmill-trigger-amqp/src/handler.rs new file mode 100644 index 0000000000..48710224df --- /dev/null +++ b/backend/windmill-trigger-amqp/src/handler.rs @@ -0,0 +1,203 @@ +use async_trait::async_trait; +use sqlx::{types::Json as SqlxJson, PgConnection}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, +}; +use windmill_git_sync::DeployedObject; +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; + +use super::{ + AmqpClientBuilder, AmqpConfig, AmqpConfigRequest, AmqpOptions, AmqpResource, AmqpTrigger, + ExchangeConfig, TestAmqpConfig, +}; + +#[async_trait] +impl TriggerCrud for AmqpTrigger { + type TriggerConfig = AmqpConfig; + type Trigger = Trigger; + type TriggerConfigRequest = AmqpConfigRequest; + type TestConnectionConfig = TestAmqpConfig; + + const TABLE_NAME: &'static str = "amqp_trigger"; + const TRIGGER_TYPE: &'static str = "amqp"; + const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = + windmill_common::user_drafts::UserDraftItemKind::TriggerAmqp; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/amqp_triggers"; + const DEPLOYMENT_NAME: &'static str = "AMQP trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = + &["amqp_resource_path", "queue_name", "exchange", "options"]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String, parent_path: Option) -> DeployedObject { + DeployedObject::AmqpTrigger { path, parent_path } + } + + async fn validate_config( + &self, + _db: &DB, + config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + if config.amqp_resource_path.trim().is_empty() { + return Err(Error::BadRequest( + "AMQP resource path cannot be empty".to_string(), + )); + } + + if config.queue_name.trim().is_empty() { + return Err(Error::BadRequest("Queue name cannot be empty".to_string())); + } + + super::validate_amqp_options(config.options.as_ref()).map_err(Error::BadRequest)?; + + Ok(()) + } + + async fn create_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let resolved_edited_by = trigger.base.resolve_edited_by(authed); + let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); + let exchange = trigger.config.exchange.map(SqlxJson); + let options = trigger.config.options.map(SqlxJson); + + sqlx::query!( + r#" + INSERT INTO amqp_trigger ( + amqp_resource_path, + queue_name, + exchange, + options, + workspace_id, + path, + script_path, + is_flow, + permissioned_as, + mode, + edited_by, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 + )"#, + trigger.config.amqp_resource_path, + trigger.config.queue_name, + exchange as Option>, + options as Option>, + w_id, + trigger.base.path, + trigger.base.script_path, + trigger.base.is_flow, + resolved_permissioned_as, + trigger.base.mode() as _, + &resolved_edited_by, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn update_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let resolved_edited_by = trigger.base.resolve_edited_by(authed); + let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed); + let exchange = trigger.config.exchange.map(SqlxJson); + let options = trigger.config.options.map(SqlxJson); + + // Important to set server_id to NULL to stop the current amqp listener + sqlx::query!( + r#" + UPDATE + amqp_trigger + SET + amqp_resource_path = $1, + queue_name = $2, + exchange = $3, + options = $4, + is_flow = $5, + edited_by = $6, + permissioned_as = $7, + script_path = $8, + path = $9, + edited_at = now(), + error = NULL, + server_id = NULL, + error_handler_path = $12, + error_handler_args = $13, + retry = $14 + WHERE + workspace_id = $10 AND + path = $11 + "#, + trigger.config.amqp_resource_path, + trigger.config.queue_name, + exchange as Option>, + options as Option>, + trigger.base.is_flow, + &resolved_edited_by, + resolved_permissioned_as, + trigger.base.script_path, + trigger.base.path, + workspace_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let amqp_resource = try_get_resource_from_db_as::( + authed, + Some(user_db.clone()), + db, + &config.amqp_resource_path, + workspace_id, + ) + .await?; + + let client_builder = AmqpClientBuilder::new(amqp_resource, "", None, None); + + client_builder + .test_connection() + .await + .map_err(|err| Error::BadConfig(format!("Error connecting to AMQP broker: {}", err)))?; + + Ok(()) + } +} diff --git a/backend/windmill-trigger-amqp/src/lib.rs b/backend/windmill-trigger-amqp/src/lib.rs new file mode 100644 index 0000000000..24ca19036d --- /dev/null +++ b/backend/windmill-trigger-amqp/src/lib.rs @@ -0,0 +1,341 @@ +use base64::engine; +use base64::prelude::*; +use lapin::{ + options::{BasicConsumeOptions, BasicQosOptions, QueueBindOptions, QueueDeclareOptions}, + types::FieldTable, + Channel, Connection, ConnectionProperties, Consumer, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow}; +use std::collections::HashMap; +use windmill_common::{error::Error, triggers::TriggerKind, worker::to_raw_value}; + +use windmill_trigger::trigger_helpers::TriggerJobArgs; + +pub mod handler; +pub mod listener; + +#[derive(Clone, Copy)] +pub struct AmqpTrigger; + +impl TriggerJobArgs for AmqpTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Amqp; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } +} + +#[derive(Debug, Deserialize)] +pub struct AmqpResource { + pub host: String, + pub port: Option, + pub username: Option, + pub password: Option, + pub vhost: Option, + pub tls: Option, +} + +/// Binding of the consumed queue to an exchange. When present, the queue is bound +/// to `exchange_name` for each routing key so messages published to the exchange +/// are routed to it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ExchangeConfig { + pub exchange_name: String, + #[serde(default)] + pub routing_keys: Vec, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AmqpOptions { + /// Declare the queue (durable) before consuming. When false the queue is + /// declared passively, i.e. it must already exist on the broker. + pub declare_queue: Option, + /// Maximum number of unacknowledged messages the broker delivers at once. + pub prefetch_count: Option, +} + +/// Shared validation for AMQP options used by both the CRUD handler and the +/// consumer builder (which also covers capture configs, that bypass CRUD +/// validation). RabbitMQ treats prefetch 0 as unlimited (unbounded consumer +/// buffer), so a set prefetch must be at least 1. +pub fn validate_amqp_options(options: Option<&AmqpOptions>) -> Result<(), String> { + if options.and_then(|o| o.prefetch_count) == Some(0) { + return Err("Prefetch count must be at least 1".to_string()); + } + Ok(()) +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct AmqpConfig { + pub amqp_resource_path: String, + pub queue_name: String, + pub exchange: Option>, + pub options: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AmqpConfigRequest { + pub amqp_resource_path: String, + pub queue_name: String, + pub exchange: Option, + pub options: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestAmqpConfig { + pub amqp_resource_path: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum AmqpError { + #[error("{0}")] + Common(#[from] Error), + #[error("{0}")] + Lapin(#[from] lapin::Error), +} + +/// Consumer bundle. The connection and channel are kept alive for as long as the +/// consumer stream is polled: dropping them would tear down the AMQP consumer. +pub struct AmqpConsumer { + pub connection: Connection, + pub channel: Channel, + pub consumer: Consumer, + pub queue_name: String, +} + +pub const CONSUMER_TAG: &str = "windmill"; + +fn build_uri(resource: &AmqpResource) -> String { + let tls = resource.tls.unwrap_or(false); + let scheme = if tls { "amqps" } else { "amqp" }; + let port = resource.port.unwrap_or(if tls { 5671 } else { 5672 }); + + let credentials = match (resource.username.as_deref(), resource.password.as_deref()) { + (Some(user), password) if !user.is_empty() => format!( + "{}:{}@", + urlencoding::encode(user), + urlencoding::encode(password.unwrap_or("")), + ), + _ => String::new(), + }; + + // The URI path is the virtual host and must be percent-encoded; the default + // vhost "/" therefore becomes "%2F". An empty vhost also falls back to "/". + let vhost = match resource.vhost.as_deref() { + Some(v) if !v.is_empty() => v, + _ => "/", + }; + let vhost_encoded = urlencoding::encode(vhost); + + // Bracket an IPv6 literal host so `host:port` parses correctly. + let host = if resource.host.contains(':') && !resource.host.starts_with('[') { + format!("[{}]", resource.host) + } else { + resource.host.clone() + }; + + format!( + "{}://{}{}:{}/{}", + scheme, credentials, host, port, vhost_encoded + ) +} + +fn connection_properties() -> ConnectionProperties { + ConnectionProperties::default() + .with_executor(tokio_executor_trait::Tokio::current()) + .with_reactor(tokio_reactor_trait::Tokio) +} + +pub struct AmqpClientBuilder<'client> { + resource: AmqpResource, + queue_name: &'client str, + exchange: Option<&'client ExchangeConfig>, + options: Option<&'client AmqpOptions>, +} + +impl<'client> AmqpClientBuilder<'client> { + pub fn new( + resource: AmqpResource, + queue_name: &'client str, + exchange: Option<&'client ExchangeConfig>, + options: Option<&'client AmqpOptions>, + ) -> Self { + Self { resource, queue_name, exchange, options } + } + + async fn connect(&self) -> Result { + let uri = build_uri(&self.resource); + let connection = Connection::connect(&uri, connection_properties()).await?; + Ok(connection) + } + + /// Establish a connection and open a channel to verify the broker is reachable + /// with the provided credentials. + pub async fn test_connection(&self) -> Result<(), AmqpError> { + let connection = self.connect().await?; + connection.create_channel().await?; + Ok(()) + } + + pub async fn build_consumer(&self) -> Result { + let connection = self.connect().await?; + let channel = connection.create_channel().await?; + + validate_amqp_options(self.options).map_err(|e| AmqpError::Common(Error::BadConfig(e)))?; + if let Some(prefetch_count) = self.options.and_then(|o| o.prefetch_count) { + channel + .basic_qos(prefetch_count, BasicQosOptions::default()) + .await?; + } + + let declare_queue = self.options.and_then(|o| o.declare_queue).unwrap_or(true); + + let queue_declare_options = QueueDeclareOptions { + passive: !declare_queue, + durable: declare_queue, + exclusive: false, + auto_delete: false, + nowait: false, + }; + + channel + .queue_declare( + self.queue_name, + queue_declare_options, + FieldTable::default(), + ) + .await?; + + if let Some(exchange) = self.exchange { + if !exchange.exchange_name.trim().is_empty() { + // Bind the queue for every routing key; an empty list binds once + // with an empty routing key (fanout exchanges ignore it anyway). + let routing_keys = if exchange.routing_keys.is_empty() { + vec![String::new()] + } else { + exchange.routing_keys.clone() + }; + for routing_key in routing_keys { + channel + .queue_bind( + self.queue_name, + &exchange.exchange_name, + &routing_key, + QueueBindOptions::default(), + FieldTable::default(), + ) + .await?; + } + } + } + + let consumer = channel + .basic_consume( + self.queue_name, + CONSUMER_TAG, + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await?; + + Ok(AmqpConsumer { connection, channel, consumer, queue_name: self.queue_name.to_string() }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn resource( + host: &str, + port: Option, + username: Option<&str>, + password: Option<&str>, + vhost: Option<&str>, + tls: Option, + ) -> AmqpResource { + AmqpResource { + host: host.to_string(), + port, + username: username.map(str::to_string), + password: password.map(str::to_string), + vhost: vhost.map(str::to_string), + tls, + } + } + + #[test] + fn build_uri_defaults_encode_root_vhost_and_pick_plaintext_port() { + let uri = build_uri(&resource("broker", None, None, None, None, None)); + assert_eq!(uri, "amqp://broker:5672/%2F"); + } + + #[test] + fn build_uri_tls_picks_amqps_and_5671() { + let uri = build_uri(&resource("broker", None, None, None, None, Some(true))); + assert_eq!(uri, "amqps://broker:5671/%2F"); + } + + #[test] + fn build_uri_encodes_credentials_and_custom_vhost() { + let uri = build_uri(&resource( + "broker", + Some(5673), + Some("us er"), + Some("p@ss/word"), + Some("my/vhost"), + None, + )); + assert_eq!(uri, "amqp://us%20er:p%40ss%2Fword@broker:5673/my%2Fvhost"); + } + + #[test] + fn build_uri_omits_credentials_when_username_empty() { + let uri = build_uri(&resource( + "broker", + None, + Some(""), + Some("secret"), + None, + None, + )); + assert_eq!(uri, "amqp://broker:5672/%2F"); + } + + #[test] + fn build_uri_blank_vhost_falls_back_to_root() { + let uri = build_uri(&resource("broker", None, None, None, Some(""), None)); + assert_eq!(uri, "amqp://broker:5672/%2F"); + } + + #[test] + fn build_uri_brackets_ipv6_host() { + let uri = build_uri(&resource("::1", Some(5672), None, None, None, None)); + assert_eq!(uri, "amqp://[::1]:5672/%2F"); + } + + fn options(prefetch: Option) -> AmqpOptions { + AmqpOptions { declare_queue: None, prefetch_count: prefetch } + } + + #[test] + fn validate_amqp_options_rejects_zero_prefetch() { + assert!(validate_amqp_options(Some(&options(Some(0)))).is_err()); + } + + #[test] + fn validate_amqp_options_accepts_valid_prefetch_and_none() { + assert!(validate_amqp_options(Some(&options(Some(1)))).is_ok()); + assert!(validate_amqp_options(Some(&options(Some(65535)))).is_ok()); + assert!(validate_amqp_options(Some(&options(None))).is_ok()); + assert!(validate_amqp_options(None).is_ok()); + } +} diff --git a/backend/windmill-trigger-amqp/src/listener.rs b/backend/windmill-trigger-amqp/src/listener.rs new file mode 100644 index 0000000000..347eb94b28 --- /dev/null +++ b/backend/windmill-trigger-amqp/src/listener.rs @@ -0,0 +1,255 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use futures::StreamExt; +use lapin::options::{BasicAckOptions, BasicNackOptions}; +use tokio::sync::RwLock; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + jobs::JobTriggerKind, + utils::{report_critical_error, report_recovered_critical_error}, + worker::to_raw_value, + DB, +}; + +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::trigger_helpers::TriggerJobArgs; +use windmill_trigger::Listener; + +use super::{AmqpClientBuilder, AmqpConfig, AmqpConsumer, AmqpResource, AmqpTrigger}; + +// lapin (like rdkafka) has no transparent reconnect, so — mirroring the Kafka +// trigger — the connection is (re)established in `consume` with a backoff retry +// loop rather than disabling the trigger on a transient broker outage. +const RECONNECT_BACKOFF_SECS: u64 = 30; +// Back off after a failed dispatch so a poison message that always fails can't +// spin a tight redelivery loop; the connection stays up for other messages. +const DISPATCH_FAILURE_BACKOFF_SECS: u64 = 5; + +impl AmqpTrigger { + async fn build_amqp_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) -> Result { + let AmqpConfig { amqp_resource_path, queue_name, exchange, options } = + &listening_trigger.trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let amqp_resource = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + db, + amqp_resource_path, + &listening_trigger.workspace_id, + ) + .await?; + + let client_builder = AmqpClientBuilder::new( + amqp_resource, + queue_name, + exchange.as_ref().map(|e| &e.0), + options.as_ref().map(|o| &o.0), + ); + + client_builder + .build_consumer() + .await + .map_err(|e| Error::BadConfig(format!("Failed to build AMQP consumer: {}", e))) + } +} + +#[async_trait] +impl Listener for AmqpTrigger { + type Consumer = (); + type Extra = (); + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Amqp; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + // The connection is established (and re-established) in `consume` so that + // a transient broker outage retries with backoff instead of disabling the + // trigger — mirroring the Kafka trigger, whose client also lacks a + // transparent reconnect. + Ok(Some(())) + } + + async fn consume( + &self, + db: &DB, + _consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + let path = &listening_trigger.path; + let workspace_id = &listening_trigger.workspace_id; + let alert_id = format!("amqp_trigger:{}", path); + let mut tries = 0_usize; + + // (Re)connect loop: retries forever with backoff; the framework's + // `select!` around `consume` cancels it on killpill, and + // `update_ping_and_loop_ping_status` returning None (trigger removed / + // disabled / capture stopped) breaks us out. + loop { + let mut consumer = match self.build_amqp_consumer(db, listening_trigger).await { + Ok(consumer) => consumer, + Err(e) => { + let status = format!( + "Failed to connect (attempt {}), retrying in {}s: {}", + tries + 1, + RECONNECT_BACKOFF_SECS, + e + ); + if self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some(status), + ) + .await + .is_none() + { + return; + } + tracing::error!( + "AMQP trigger {} failed to connect (attempt {}): {}", + path, + tries + 1, + e + ); + if tries % 10 == 0 && listening_trigger.trigger_mode { + report_critical_error( + format!( + "Failed to connect AMQP trigger {} (attempt {}), retrying every {}s. This alert repeats every 10 failed attempts. Error: {}", + path, tries + 1, RECONNECT_BACKOFF_SECS, e + ), + db.clone(), + Some(workspace_id), + Some(&alert_id), + ) + .await; + } + tries += 1; + tokio::time::sleep(Duration::from_secs(RECONNECT_BACKOFF_SECS)).await; + continue; + } + }; + + // Connected: clear any "reconnecting" status. + if self + .update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), None) + .await + .is_none() + { + return; + } + if tries > 0 { + tracing::info!("AMQP trigger {} reconnected after {} attempts", path, tries); + if listening_trigger.trigger_mode { + report_recovered_critical_error( + format!("AMQP trigger {} reconnected", path), + db.clone(), + Some(workspace_id), + Some(&alert_id), + ) + .await; + } + tries = 0; + } + + // Consume until the stream errors, then break out to reconnect. + loop { + match consumer.consumer.next().await { + Some(Ok(delivery)) => { + let trigger_info = HashMap::from([ + ( + "exchange".to_string(), + to_raw_value(&delivery.exchange.as_str()), + ), + ( + "routing_key".to_string(), + to_raw_value(&delivery.routing_key.as_str()), + ), + ("queue_name".to_string(), to_raw_value(&consumer.queue_name)), + ( + "redelivered".to_string(), + to_raw_value(&delivery.redelivered), + ), + ( + "delivery_tag".to_string(), + to_raw_value(&delivery.delivery_tag), + ), + ]); + + let dispatched = self + .handle_event( + db, + listening_trigger, + delivery.data.clone(), + trigger_info, + None, + ) + .await; + + // Only ack once the job/capture was dispatched. On failure + // nack with requeue so the broker redelivers rather than + // dropping the message (at-least-once). + let dispatch_failed = dispatched.is_err(); + let ack_result = if dispatch_failed { + delivery + .acker + .nack(BasicNackOptions { requeue: true, multiple: false }) + .await + } else { + delivery.acker.ack(BasicAckOptions::default()).await + }; + + if let Err(err) = ack_result { + // Channel is gone; break out to reconnect. + tracing::warn!( + "AMQP trigger {} ack/nack failed, reconnecting: {}", + path, + err + ); + break; + } + + if dispatch_failed { + // The message was requeued: back off before consuming + // again so a poison message can't spin a tight + // redelivery loop, while keeping the connection alive. + tokio::time::sleep(Duration::from_secs(DISPATCH_FAILURE_BACKOFF_SECS)) + .await; + } + } + Some(Err(err)) => { + tracing::warn!( + "AMQP trigger {} consumer error, reconnecting: {}", + path, + err + ); + break; + } + None => { + tracing::warn!("AMQP trigger {} consumer stream ended, reconnecting", path); + break; + } + } + } + } + } +} diff --git a/backend/windmill-trigger-postgres/src/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs index 10a26e0066..0592c48bf7 100644 --- a/backend/windmill-trigger-postgres/src/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -20,7 +20,7 @@ use windmill_common::{ }; use windmill_git_sync::DeployedObject; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{check_scopes, ApiAuthed}; use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ @@ -46,7 +46,8 @@ impl TriggerCrud for PostgresTrigger { const TABLE_NAME: &'static str = "postgres_trigger"; const TRIGGER_TYPE: &'static str = "postgres"; - const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = windmill_common::user_drafts::UserDraftItemKind::TriggerPostgres; + const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind = + windmill_common::user_drafts::UserDraftItemKind::TriggerPostgres; const SUPPORTS_SERVER_STATE: bool = true; const SUPPORTS_TEST_CONNECTION: bool = true; const ROUTE_PREFIX: &'static str = "/postgres_triggers"; @@ -395,6 +396,10 @@ pub async fn get_postgres_version( Extension(user_db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:read:{}", postgres_resource_path) + })?; + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db), @@ -416,6 +421,10 @@ pub async fn list_slot_name( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { + check_scopes(&authed, || { + format!("postgres_triggers:read:{}", postgres_resource_path) + })?; + let pg_connection: Client = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -458,6 +467,10 @@ pub async fn create_slot( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -513,6 +526,10 @@ pub async fn drop_slot_name( Path((w_id, postgres_resource_path)): Path<(String, String)>, Json(Slot { name }): Json, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let pg_connection = get_default_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id) .await @@ -531,6 +548,10 @@ pub async fn list_database_publication( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> Result>> { + check_scopes(&authed, || { + format!("postgres_triggers:read:{}", postgres_resource_path) + })?; + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -563,6 +584,10 @@ pub async fn get_publication_info( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result> { + check_scopes(&authed, || { + format!("postgres_triggers:read:{}", postgres_resource_path) + })?; + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -604,6 +629,10 @@ pub async fn create_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -640,6 +669,10 @@ pub async fn delete_publication( Extension(db): Extension, Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -773,6 +806,10 @@ pub async fn alter_publication( Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, Json(publication_data): Json, ) -> Result { + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let mut pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), @@ -948,6 +985,10 @@ pub async fn create_template_script( ) -> Result { let TemplateScript { postgres_resource_path, relations, language } = template_script; + check_scopes(&authed, || { + format!("postgres_triggers:write:{}", postgres_resource_path) + })?; + let relations = match relations { Some(r) => r, None => { @@ -1090,6 +1131,10 @@ pub async fn is_database_in_logical_level( Extension(db): Extension, Path((w_id, postgres_resource_path)): Path<(String, String)>, ) -> error::JsonResult { + check_scopes(&authed, || { + format!("postgres_triggers:read:{}", postgres_resource_path) + })?; + let pg_connection = get_default_pg_connection( authed.clone(), Some(user_db.clone()), diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index d6fcebfddb..a295183dfc 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use serde_json::value::RawValue; use sqlx::FromRow; use windmill_api_auth::ApiAuthed; -use windmill_common::workspaces::get_datatable_resource_from_db_unchecked; +use windmill_common::workspaces::get_datatable_replication_resource_from_db_unchecked; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, @@ -382,8 +382,10 @@ pub async fn resolve_postgres_resource( w_id: &str, ) -> Result { if let Some(datatable_name) = postgres_resource_path.strip_prefix("datatable://") { + // Trigger connections (publication/slot management + logical replication) run + // as the dedicated replication user on custom-instance databases. let resource_value = - get_datatable_resource_from_db_unchecked(db, w_id, datatable_name).await?; + get_datatable_replication_resource_from_db_unchecked(db, w_id, datatable_name).await?; serde_json::from_value::(resource_value).map_err(|e| Error::SerdeJson { error: e, location: "resolve_postgres_resource".to_string(), diff --git a/backend/windmill-trigger-websocket/src/handler.rs b/backend/windmill-trigger-websocket/src/handler.rs index 529e5902cc..c2ca37c500 100644 --- a/backend/windmill-trigger-websocket/src/handler.rs +++ b/backend/windmill-trigger-websocket/src/handler.rs @@ -316,9 +316,9 @@ impl TriggerCrud for WebsocketTrigger { Cow::Borrowed(&url) }; - validate_websocket_url_for_ssrf(&connect_url).await?; + let validated = validate_websocket_url_for_ssrf(&connect_url).await?; - connect_async_with_proxy(&*connect_url) + connect_async_with_proxy(&*connect_url, validated.pinned_addrs()) .await .map_err(|err| { Error::BadConfig(format!( diff --git a/backend/windmill-trigger-websocket/src/lib.rs b/backend/windmill-trigger-websocket/src/lib.rs index 4111606b2b..81b87ee3a1 100644 --- a/backend/windmill-trigger-websocket/src/lib.rs +++ b/backend/windmill-trigger-websocket/src/lib.rs @@ -119,14 +119,15 @@ pub const ALLOW_PRIVATE_WEBSOCKET_URLS_ENV: &str = "ALLOW_PRIVATE_WEBSOCKET_URLS /// `$flow:`/`$script:` URL is checked on its returned value and re-checked on /// each reconnect (DNS rebinding). `validate_config` also calls this at save /// time to reject static URLs early. -pub async fn validate_websocket_url_for_ssrf(url: &str) -> Result<()> { - if std::env::var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV) - .ok() - .is_some_and(|v| v == "true" || v == "1") - { - return Ok(()); - } - +/// +/// Returns the resolved [`ValidatedTarget`]: the connect must pin these +/// addresses (see [`proxy::connect_async_with_proxy`]) so a rebinder cannot swap +/// in an internal IP between this check and the connect (TOCTOU). The returned +/// `addrs` carry the http(s) port, which equals the ws(s) port the connection +/// uses (ws→80, wss→443, or the explicit port preserved through the mapping). +pub async fn validate_websocket_url_for_ssrf( + url: &str, +) -> Result { // `ws`/`wss` aren't recognised by `validate_url_for_ssrf`'s scheme check, so // map them to the http(s) equivalent the same connection would tunnel over. // The prefixes are ASCII, so byte-slicing at their length stays on a char @@ -140,6 +141,20 @@ pub async fn validate_websocket_url_for_ssrf(url: &str) -> Result<()> { url.to_string() }; + if std::env::var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") + { + // Opted out of the SSRF guard: allow any host and pin nothing. Parse the + // host for a uniform return; fall back to an empty target if unparseable + // (the connect then resolves normally, matching the pre-guard behavior). + let host = url::Url::parse(&http_url) + .ok() + .and_then(|u| u.host_str().map(str::to_string)) + .unwrap_or_default(); + return Ok(windmill_common::ssrf::ValidatedTarget { host, addrs: Vec::new() }); + } + windmill_common::ssrf::validate_url_for_ssrf(&http_url) .await .map_err(|e| match e { diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index e34fe6e4f6..a38f2434a4 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -191,7 +191,7 @@ impl Listener for WebsocketTrigger { Cow::Borrowed(&url) }; - validate_websocket_url_for_ssrf(&connect_url).await?; + let validated = validate_websocket_url_for_ssrf(&connect_url).await?; // Gateway endpoints are often fronted by an edge proxy (e.g. Cloudflare) // that sporadically answers the upgrade request with a transient 5xx @@ -203,7 +203,7 @@ impl Listener for WebsocketTrigger { let mut attempt = 0; loop { attempt += 1; - match connect_async_with_proxy(&*connect_url).await { + match connect_async_with_proxy(&*connect_url, validated.pinned_addrs()).await { Ok(conn) => return Ok(Some(conn)), // Only retry in trigger mode: a failed connect there disables the // trigger until a human re-enables it, while capture mode is an diff --git a/backend/windmill-trigger-websocket/src/proxy.rs b/backend/windmill-trigger-websocket/src/proxy.rs index 8670b7ca3a..701401d277 100644 --- a/backend/windmill-trigger-websocket/src/proxy.rs +++ b/backend/windmill-trigger-websocket/src/proxy.rs @@ -15,6 +15,7 @@ use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use std::io; +use std::net::SocketAddr; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, net::TcpStream, @@ -33,19 +34,28 @@ use windmill_common::{HTTPS_PROXY, HTTP_PROXY, NO_PROXY}; /// Drop-in replacement for `tokio_tungstenite::connect_async` that routes /// the underlying TCP connection through `HTTPS_PROXY` / `HTTP_PROXY` -/// (with `NO_PROXY` exclusions) when those env vars are set. When none -/// is set we short-circuit straight to `connect_async`, keeping the -/// behaviour for non-proxied deployments unchanged. +/// (with `NO_PROXY` exclusions) when those env vars are set, and — for direct +/// (non-proxied) connections — pins DNS to `pinned_addrs`. +/// +/// `pinned_addrs` are the addresses the SSRF guard already resolved and +/// validated for this URL (see `validate_websocket_url_for_ssrf`). Connecting +/// straight to them, rather than letting `connect_async` re-resolve the host, +/// closes the DNS-rebinding window between the check and the connect: a rebinder +/// cannot answer a public IP at validation time and an internal one here. When +/// `pinned_addrs` is empty (IP-literal host, or the SSRF guard opted out via +/// `ALLOW_PRIVATE_WEBSOCKET_URLS`) there is nothing to pin and we fall back to +/// `connect_async`, keeping the behaviour for those cases unchanged. +/// +/// When a proxy applies, the proxy itself resolves the target host, so DNS +/// rebinding at this hop is not the worker's concern and `pinned_addrs` is +/// unused for that path. pub async fn connect_async_with_proxy( request: R, + pinned_addrs: &[SocketAddr], ) -> Result<(WebSocketStream>, Response), WsError> where R: IntoClientRequest + Unpin, { - if HTTPS_PROXY.is_none() && HTTP_PROXY.is_none() { - return connect_async(request).await; - } - let request = request.into_client_request()?; let uri = request.uri().clone(); let scheme = uri.scheme_str().unwrap_or_default().to_ascii_lowercase(); @@ -62,26 +72,49 @@ where }) .ok_or(WsError::Url(UrlError::UnsupportedUrlScheme))?; - let proxy = proxy_url_for(&scheme, &host).and_then(|raw| parse_proxy_target(&raw)); - - let Some(proxy) = proxy else { - // Proxy env was set but doesn't apply to this host (NO_PROXY hit - // or unparseable URL): preserve the original connect path. - return connect_async(request).await; + let proxy = if HTTPS_PROXY.is_none() && HTTP_PROXY.is_none() { + None + } else { + proxy_url_for(&scheme, &host).and_then(|raw| parse_proxy_target(&raw)) }; - tracing::debug!( - "Connecting to WebSocket {}:{} through HTTP proxy {}:{}", - host, - port, - proxy.host, - proxy.port, - ); - let socket = http_connect_tunnel(&proxy, &host, port) - .await - .map_err(WsError::Io)?; + if let Some(proxy) = proxy { + tracing::debug!( + "Connecting to WebSocket {}:{} through HTTP proxy {}:{}", + host, + port, + proxy.host, + proxy.port, + ); + let socket = http_connect_tunnel(&proxy, &host, port) + .await + .map_err(WsError::Io)?; + return client_async_tls_with_config(request, socket, None, None).await; + } - client_async_tls_with_config(request, socket, None, None).await + // Direct connection. Nothing to pin (IP literal or SSRF guard opted out): + // preserve the original resolve-and-connect path. + if pinned_addrs.is_empty() { + return connect_async(request).await; + } + + // Pin to a validated address so this connect targets the same IP the SSRF + // guard checked. Try each in order (e.g. IPv6 then IPv4) until one connects. + let mut last_err: Option = None; + for addr in pinned_addrs { + match TcpStream::connect(addr).await { + Ok(socket) => { + return client_async_tls_with_config(request, socket, None, None).await; + } + Err(e) => last_err = Some(e), + } + } + Err(WsError::Io(last_err.unwrap_or_else(|| { + io::Error::new( + io::ErrorKind::AddrNotAvailable, + "no pinned address to connect", + ) + }))) } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index ed3b89fc65..0c46ff25fd 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -6,13 +6,13 @@ * LICENSE-AGPL for a copy of the license. */ -use crate::types::{StandardTriggerQuery, TriggerData, TriggerMode}; +use crate::types::{HasPath, StandardTriggerQuery, TriggerData, TriggerMode}; use async_trait::async_trait; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{FromRow, PgConnection}; use std::fmt::Debug; -use windmill_api_auth::{check_scopes, ApiAuthed}; +use windmill_api_auth::{build_scope_path_predicate, check_scopes, ApiAuthed}; use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, @@ -65,6 +65,8 @@ pub trait TriggerCrud: Send + Sync + 'static { + Send + Sync + Unpin + // Path accessor so `list_triggers` can apply scoped-token filtering. + + HasPath // `'static` so the deployed trigger can be boxed into // `WithDraftOverlay`'s erased-serde inner (it's an owned row). + 'static; @@ -662,6 +664,9 @@ async fn list_triggers( } } + let allowed = build_scope_path_predicate(&authed, T::scope_domain_name(), "read"); + triggers.retain(|t| allowed(t.trigger_path())); + Ok(Json(triggers)) } @@ -715,6 +720,12 @@ async fn update_trigger( Json(mut edit_trigger): Json>, ) -> Result { let path = path.to_path(); + // A scoped token must be allowed on both the existing path (URL) and the + // new path (body); checking only the latter would let it move a trigger it + // can't touch into its scope. + check_scopes(&authed, || { + format!("{}:write:{}", T::scope_domain_name(), &path) + })?; check_scopes(&authed, || { format!( "{}:write:{}", diff --git a/backend/windmill-trigger/src/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs index 1427f7f102..e0bfaf3d34 100644 --- a/backend/windmill-trigger/src/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -140,7 +140,7 @@ fn runnable_format_from_schema_without_preprocessor( schema: Option>, ) -> RunnableFormat { match trigger_kind { - TriggerKind::Mqtt + TriggerKind::Mqtt | TriggerKind::Amqp if schema.as_ref().is_some_and(|schema| { schema.properties.as_ref().is_some_and(|properties| { properties.iter().any(|(key, def)| { diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs index 77d90bea20..dcf9a136be 100644 --- a/backend/windmill-trigger/src/types.rs +++ b/backend/windmill-trigger/src/types.rs @@ -102,6 +102,28 @@ where pub error_handling: TriggerErrorHandling, } +/// Path accessor for the associated `Trigger` row type, so the shared list +/// handler can apply scoped-token path filtering without knowing the concrete +/// row shape. The `()` OSS stub returns "" (its endpoints 404 anyway). +pub trait HasPath { + fn trigger_path(&self) -> &str; +} + +impl HasPath for Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + fn trigger_path(&self) -> &str { + &self.base.path + } +} + +impl HasPath for () { + fn trigger_path(&self) -> &str { + "" + } +} + impl FromRow<'_, sqlx::postgres::PgRow> for Trigger where T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, diff --git a/backend/windmill-types/src/flow_status.rs b/backend/windmill-types/src/flow_status.rs index 62da4d5517..79890e3c6e 100644 --- a/backend/windmill-types/src/flow_status.rs +++ b/backend/windmill-types/src/flow_status.rs @@ -475,6 +475,33 @@ impl FlowStatusModule { } } + /// For a still-`InProgress` module (a between-steps zombie), whether the module's own + /// iteration/branch cursor proves it actually reached the end, so the only thing left is + /// the final state transition (children-success is a separate, DB-side check). + /// + /// A serial for-loop / branch-all grows `flow_jobs` one entry at a time, so an all-success + /// prefix does NOT mean the module finished: the cursor must sit on the last element. Parallel + /// containers preallocate every child up front, so a full success set is conclusive. While-loops + /// are never derivable here (continuation depends on a condition evaluated after each iteration, + /// which a reaped zombie never persisted). Non-`InProgress` modules return false. + pub fn is_between_steps_complete(&self) -> bool { + match self { + FlowStatusModule::InProgress { while_loop: true, .. } => false, + // Parallel loop/branch-all: all children exist up front, so children-success suffices. + FlowStatusModule::InProgress { parallel: true, .. } => true, + FlowStatusModule::InProgress { iterator: Some(it), .. } => { + let total = it + .itered_len + .or_else(|| it.itered.as_ref().map(|v| v.len())); + total.is_some_and(|t| t > 0 && it.index + 1 == t) + } + FlowStatusModule::InProgress { branchall: Some(ba), .. } => ba.branch + 1 == ba.len, + // Single-child leaf / subflow / branch-one: the child ran, nothing else to advance. + FlowStatusModule::InProgress { .. } => true, + _ => false, + } + } + pub fn agent_actions(&self) -> Option> { match self { FlowStatusModule::InProgress { agent_actions, .. } => agent_actions.clone(), @@ -549,4 +576,131 @@ impl FlowStatus { let i = usize::try_from(self.step).ok()?; self.modules.get(i) } + + /// Whether no step has begun executing yet: the preprocessor (if any) and the first + /// module are both still `WaitingForPriorSteps`. A reaped flow in this state can be + /// safely re-queued because nothing ran. A preprocessor that is `InProgress` means its + /// child already ran (only the parent transition was lost), so re-queuing would + /// re-run the preprocessor and duplicate its side effects. + pub fn is_not_yet_started(&self) -> bool { + self.preprocessor_module + .as_ref() + .is_none_or(|p| matches!(p, FlowStatusModule::WaitingForPriorSteps { .. })) + && self + .modules + .first() + .is_some_and(|m| matches!(m, FlowStatusModule::WaitingForPriorSteps { .. })) + } +} + +#[cfg(test)] +mod tests { + use super::{FlowStatus, FlowStatusModule}; + + fn module(json: serde_json::Value) -> FlowStatusModule { + serde_json::from_value(json).unwrap() + } + + fn status(json: serde_json::Value) -> FlowStatus { + serde_json::from_value(json).unwrap() + } + + #[test] + fn is_not_yet_started_distinguishes_preprocessor_zombie() { + let nil = "00000000-0000-0000-0000-000000000000"; + let waiting = serde_json::json!({ "type": "WaitingForPriorSteps", "id": "a" }); + let failure = serde_json::json!({ "type": "WaitingForPriorSteps", "id": "failure" }); + // No preprocessor, first module waiting: genuinely unstarted. + assert!(status(serde_json::json!({ + "step": 0, "modules": [waiting], "failure_module": failure + })) + .is_not_yet_started()); + // First module already InProgress: started. + assert!(!status(serde_json::json!({ + "step": 0, + "modules": [{ "type": "InProgress", "id": "a", "job": nil }], + "failure_module": failure + })) + .is_not_yet_started()); + // Preprocessor still waiting, first module waiting: unstarted. + assert!(status(serde_json::json!({ + "step": -1, "modules": [waiting], "failure_module": failure, + "preprocessor_module": { "type": "WaitingForPriorSteps", "id": "pre" } + })) + .is_not_yet_started()); + // Preprocessor InProgress (its child ran) while modules[0] still waits: a + // preprocessor zombie, NOT unstarted, so it must not be auto-requeued. + assert!(!status(serde_json::json!({ + "step": -1, "modules": [waiting], "failure_module": failure, + "preprocessor_module": { "type": "InProgress", "id": "pre", "job": nil } + })) + .is_not_yet_started()); + } + + #[test] + fn between_steps_complete_serial_loop() { + // Cursor on the last iteration => complete. + assert!(module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "iterator": { "index": 1, "itered_len": 2 }, "flow_jobs": [] + })) + .is_between_steps_complete()); + // Reaped mid-iteration (iteration 1 of 2 never scheduled) => NOT complete. + assert!(!module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "iterator": { "index": 0, "itered_len": 2 }, "flow_jobs": [] + })) + .is_between_steps_complete()); + // Legacy shape: itered array present, itered_len absent. + assert!(module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "iterator": { "index": 1, "itered": ["x", "y"] } + })) + .is_between_steps_complete()); + } + + #[test] + fn between_steps_complete_while_loop_never() { + assert!(!module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "while_loop": true, "iterator": { "index": 1, "itered_len": 2 } + })) + .is_between_steps_complete()); + } + + #[test] + fn between_steps_complete_branchall_and_parallel() { + // Serial branch-all on the last branch => complete; earlier branch => not. + assert!(module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "branchall": { "branch": 1, "len": 2 } + })) + .is_between_steps_complete()); + assert!(!module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "branchall": { "branch": 0, "len": 2 } + })) + .is_between_steps_complete()); + // Parallel loop: children preallocated, so any cursor is fine (success is checked elsewhere). + assert!(module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "parallel": true, "iterator": { "index": 0, "itered_len": 3 } + })) + .is_between_steps_complete()); + } + + #[test] + fn between_steps_complete_leaf_and_non_inprogress() { + // Single-child leaf: the child ran, nothing to advance. + assert!(module(serde_json::json!({ + "type": "InProgress", "id": "a", "job": "00000000-0000-0000-0000-000000000000" + })) + .is_between_steps_complete()); + // A Success module is not a between-steps zombie. + assert!(!module(serde_json::json!({ + "type": "Success", "id": "a", "job": "00000000-0000-0000-0000-000000000000", + "skipped": false + })) + .is_between_steps_complete()); + } } diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 0c95b9612b..b06e2af05e 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -145,6 +145,57 @@ impl NewFlow { } } +/// Body for updating an existing flow. Mirrors `NewFlow`, but `path` is optional: the +/// flow to update is identified by the URL, so the body only needs `path` to rename it. +/// This matches the `EditVariable` / `EditResource` / `EditApp` convention and lets a +/// caller update in place without restating the path. +#[derive(Debug, Deserialize)] +pub struct EditFlow { + #[serde(default)] + pub path: Option, + pub summary: String, + pub description: Option, + #[serde(deserialize_with = "validate_flow_value")] + pub value: Box, + pub schema: Option, + pub tag: Option, + pub dedicated_worker: Option, + pub timeout: Option, + pub deployment_message: Option, + pub visible_to_runner_only: Option, + pub on_behalf_of_email: Option, + pub preserve_on_behalf_of: Option, + pub ws_error_handler_muted: Option, + #[serde(default)] + pub labels: Option>, + #[serde(default)] + pub skip_draft_deletion: Option, +} + +impl EditFlow { + /// Resolve into a `NewFlow`, defaulting the target path to `current_path` (the flow's + /// URL path) when the body omits it. A body `path` that differs renames the flow. + pub fn into_new_flow(self, current_path: &str) -> NewFlow { + NewFlow { + path: self.path.unwrap_or_else(|| current_path.to_string()), + summary: self.summary, + description: self.description, + value: self.value, + schema: self.schema, + tag: self.tag, + dedicated_worker: self.dedicated_worker, + timeout: self.timeout, + deployment_message: self.deployment_message, + visible_to_runner_only: self.visible_to_runner_only, + on_behalf_of_email: self.on_behalf_of_email, + preserve_on_behalf_of: self.preserve_on_behalf_of, + ws_error_handler_muted: self.ws_error_handler_muted, + labels: self.labels, + skip_draft_deletion: self.skip_draft_deletion, + } + } +} + fn validate_retry(retry: &Retry, module_id: &str) -> anyhow::Result<()> { if retry.exponential.attempts > 0 && retry.exponential.seconds == 0 { return Err(anyhow::anyhow!( @@ -613,6 +664,19 @@ impl FlowModule { .is_ok_and(|x| x == "script" || x == "rawscript" || x == "flowscript") } + /// Whether a between-steps-zombie step carrying this definition can be safely reused as + /// `Success` on restart (see restart-resolution reuse). Excludes steps whose completion + /// transition or arming carries semantics that reuse would silently skip: stop predicates + /// (`stop_after_if` / `stop_after_all_iters_if`, which decide whether downstream steps run), + /// `skip_if` (skipped-state and suspend arming), a `suspend` approval boundary, and `sleep`. + pub fn allows_zombie_reuse(&self) -> bool { + self.stop_after_if.is_none() + && self.stop_after_all_iters_if.is_none() + && self.skip_if.is_none() + && self.suspend.is_none() + && self.sleep.is_none() + } + pub fn get_type(&self) -> anyhow::Result<&str> { #[derive(Deserialize)] pub struct FlowModuleValueType<'a> { @@ -1233,6 +1297,20 @@ mod tests { use super::*; use serde_json::json; + #[test] + fn edit_flow_defaults_path_from_url_and_renames_when_given() { + // An omitted body path resolves to the URL path; an explicit body path renames. + let ef: EditFlow = + serde_json::from_value(json!({ "summary": "s", "value": { "modules": [] } })).unwrap(); + assert_eq!(ef.into_new_flow("f/team/my_flow").path, "f/team/my_flow"); + + let ef: EditFlow = serde_json::from_value( + json!({ "path": "f/team/renamed", "summary": "s", "value": { "modules": [] } }), + ) + .unwrap(); + assert_eq!(ef.into_new_flow("f/team/my_flow").path, "f/team/renamed"); + } + #[test] fn flow_value_ignores_notes_and_groups() { // FlowValue should parse successfully even when notes/groups are present — diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 95ac69bc2c..79b33f86ac 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -32,6 +32,7 @@ pub enum JobTriggerKind { Email, Nats, Mqtt, + Amqp, Sqs, Postgres, Schedule, @@ -66,6 +67,7 @@ impl std::fmt::Display for JobTriggerKind { JobTriggerKind::Email => "email", JobTriggerKind::Nats => "nats", JobTriggerKind::Mqtt => "mqtt", + JobTriggerKind::Amqp => "amqp", JobTriggerKind::Sqs => "sqs", JobTriggerKind::Postgres => "postgres", JobTriggerKind::Schedule => "schedule", diff --git a/backend/windmill-types/src/runnable_settings.rs b/backend/windmill-types/src/runnable_settings.rs index ea0fd30dc0..038246cbe6 100644 --- a/backend/windmill-types/src/runnable_settings.rs +++ b/backend/windmill-types/src/runnable_settings.rs @@ -93,9 +93,7 @@ pub struct DebouncingSettings { pub debounce_args_to_accumulate: Option>, } -#[derive( - Debug, Default, Clone, Serialize, Deserialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode, -)] +#[derive(Debug, Default, Clone, Serialize, Hash, PartialEq, sqlx::FromRow, sqlx::Decode)] pub struct ConcurrencySettings { #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_key: Option, @@ -105,7 +103,65 @@ pub struct ConcurrencySettings { pub concurrency_time_window_s: Option, } -#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] +/// Shared normalization for the positive-only `Option` runnable settings +/// (`concurrent_limit`, `timeout`, ...): a `<= 0` value is never meaningful — zero +/// concurrent slots permanently blocks a runnable at the concurrency gate (a re-queue +/// storm the zombie monitor eventually fails with a misleading OOM error), and a +/// 0-second timeout kills every job on the spot. The frontend already treats `0` as +/// "disabled", so `<= 0` maps to `None` (unset) everywhere. Idempotent. +pub fn none_if_non_positive(v: Option) -> Option { + v.filter(|n| *n > 0) +} + +/// Coerce a `concurrent_limit <= 0` to disabled, dropping the now-meaningless time window +/// alongside it. Idempotent. +fn normalize_concurrency( + concurrent_limit: &mut Option, + concurrency_time_window_s: &mut Option, +) { + if none_if_non_positive(*concurrent_limit).is_none() { + *concurrent_limit = None; + *concurrency_time_window_s = None; + } +} + +impl ConcurrencySettings { + pub fn normalized(mut self) -> Self { + normalize_concurrency( + &mut self.concurrent_limit, + &mut self.concurrency_time_window_s, + ); + self + } +} + +// Manual `Deserialize` so every ingestion path (script/flow create & update, app and +// http-trigger payloads, and read-back of already-stored settings) normalizes a `<= 0` +// limit uniformly, without each call site remembering to call `normalized()`. +impl<'de> Deserialize<'de> for ConcurrencySettings { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct Raw { + #[serde(default)] + concurrency_key: Option, + #[serde(default)] + concurrent_limit: Option, + #[serde(default)] + concurrency_time_window_s: Option, + } + let Raw { concurrency_key, concurrent_limit, concurrency_time_window_s } = + Raw::deserialize(deserializer)?; + Ok( + ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s } + .normalized(), + ) + } +} + +#[derive(Debug, Clone, Serialize, sqlx::FromRow, Default)] pub struct ConcurrencySettingsWithCustom { #[serde(skip_serializing_if = "Option::is_none")] pub custom_concurrency_key: Option, @@ -115,6 +171,41 @@ pub struct ConcurrencySettingsWithCustom { pub concurrency_time_window_s: Option, } +impl ConcurrencySettingsWithCustom { + pub fn normalized(mut self) -> Self { + normalize_concurrency( + &mut self.concurrent_limit, + &mut self.concurrency_time_window_s, + ); + self + } +} + +impl<'de> Deserialize<'de> for ConcurrencySettingsWithCustom { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct Raw { + #[serde(default)] + custom_concurrency_key: Option, + #[serde(default)] + concurrent_limit: Option, + #[serde(default)] + concurrency_time_window_s: Option, + } + let Raw { custom_concurrency_key, concurrent_limit, concurrency_time_window_s } = + Raw::deserialize(deserializer)?; + Ok(ConcurrencySettingsWithCustom { + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + } + .normalized()) + } +} + impl DebouncingSettings { pub fn maybe_fallback( self, @@ -142,11 +233,15 @@ impl ConcurrencySettings { concurrent_limit: Option, concurrency_time_window_s: Option, ) -> Self { + // Legacy columns can still hold a stored `0` that predates ingestion normalization, + // so re-normalize here: this is the single load boundary for every DB-backed read + // (script/schedule read, flow value, and the worker pull path). Self { concurrency_key: self.concurrency_key.or(concurrency_key), concurrent_limit: self.concurrent_limit.or(concurrent_limit), concurrency_time_window_s: self.concurrency_time_window_s.or(concurrency_time_window_s), } + .normalized() } } @@ -229,4 +324,91 @@ mod tests { assert_eq!(r, Retry::default()); assert_eq!(r.exponential.multiplier, 1); } + + // The positive-only settings share one rule: `<= 0` means "unset". This is what keeps a + // stored `0` from being enforced as a zero-slot cap or a 0-second timeout. + #[test] + fn none_if_non_positive_coerces_zero_and_negative() { + assert_eq!(none_if_non_positive(Some(0)), None); + assert_eq!(none_if_non_positive(Some(-3)), None); + assert_eq!(none_if_non_positive(Some(1)), Some(1)); + assert_eq!(none_if_non_positive(Some(i32::MAX)), Some(i32::MAX)); + assert_eq!(none_if_non_positive(None), None); + } + + // Ingestion path (scripts flatten this on `NewScript`, flows on `FlowModule`): a `0` + // concurrent_limit deserializes to disabled and drops the now-meaningless time window, + // while a real limit and its window survive untouched. + #[test] + fn concurrency_settings_deserialize_normalizes_non_positive_limit() { + let zero: ConcurrencySettings = serde_json::from_value( + serde_json::json!({"concurrent_limit": 0, "concurrency_time_window_s": 30}), + ) + .unwrap(); + assert_eq!(zero.concurrent_limit, None); + assert_eq!(zero.concurrency_time_window_s, None); + + let negative: ConcurrencySettings = + serde_json::from_value(serde_json::json!({"concurrent_limit": -1})).unwrap(); + assert_eq!(negative.concurrent_limit, None); + + let real: ConcurrencySettings = serde_json::from_value( + serde_json::json!({"concurrent_limit": 2, "concurrency_time_window_s": 30}), + ) + .unwrap(); + assert_eq!(real.concurrent_limit, Some(2)); + assert_eq!(real.concurrency_time_window_s, Some(30)); + } + + // Per-flow-step overrides use the `custom_concurrency_key` variant; same rule. + #[test] + fn concurrency_settings_with_custom_deserialize_normalizes() { + let zero: ConcurrencySettingsWithCustom = serde_json::from_value( + serde_json::json!({"concurrent_limit": 0, "concurrency_time_window_s": 5}), + ) + .unwrap(); + assert_eq!(zero.concurrent_limit, None); + assert_eq!(zero.concurrency_time_window_s, None); + } + + // A normalized value serializes with the limit omitted (skip_serializing_if), matching the + // frontend's "disabled" representation instead of re-emitting a `0`. + #[test] + fn normalized_disabled_limit_serializes_as_omitted() { + let s = + ConcurrencySettings { concurrent_limit: Some(0), ..Default::default() }.normalized(); + let json = serde_json::to_value(&s).unwrap(); + assert!(json.get("concurrent_limit").is_none()); + } + + // Runtime load boundary: legacy rows still hold a raw `0` in the fallback columns. The + // fallback must not resurrect it as an active limit. + #[test] + fn maybe_fallback_normalizes_legacy_zero_column() { + let merged = ConcurrencySettings::default().maybe_fallback(None, Some(0), Some(30)); + assert_eq!(merged.concurrent_limit, None); + assert_eq!(merged.concurrency_time_window_s, None); + } + + // `NewScript`/`FlowModule` embed the settings via `#[serde(flatten)]`, which drives the + // manual Deserialize through a content-buffer deserializer rather than a plain map. Guard + // that path: normalization must still fire and sibling fields must still parse. + #[test] + fn flattened_concurrency_normalizes_and_preserves_siblings() { + #[derive(Deserialize)] + struct Wrapper { + name: String, + #[serde(flatten)] + concurrency: ConcurrencySettings, + } + let w: Wrapper = serde_json::from_value(serde_json::json!({ + "name": "s", + "concurrent_limit": 0, + "concurrency_time_window_s": 42, + })) + .unwrap(); + assert_eq!(w.name, "s"); + assert_eq!(w.concurrency.concurrent_limit, None); + assert_eq!(w.concurrency.concurrency_time_window_s, None); + } } diff --git a/backend/windmill-types/src/triggers.rs b/backend/windmill-types/src/triggers.rs index 8ad1d5e8a7..e7c398b1f2 100644 --- a/backend/windmill-types/src/triggers.rs +++ b/backend/windmill-types/src/triggers.rs @@ -16,6 +16,7 @@ pub enum TriggerKind { Email, Nats, Mqtt, + Amqp, Sqs, Postgres, Gcp, @@ -36,6 +37,7 @@ impl TriggerKind { TriggerKind::DefaultEmail => "email".to_string(), TriggerKind::Nats => "nats".to_string(), TriggerKind::Mqtt => "mqtt".to_string(), + TriggerKind::Amqp => "amqp".to_string(), TriggerKind::Sqs => "sqs".to_string(), TriggerKind::Postgres => "postgres".to_string(), TriggerKind::Gcp => "gcp".to_string(), @@ -58,6 +60,7 @@ impl fmt::Display for TriggerKind { TriggerKind::DefaultEmail => "default_email", TriggerKind::Nats => "nats", TriggerKind::Mqtt => "mqtt", + TriggerKind::Amqp => "amqp", TriggerKind::Sqs => "sqs", TriggerKind::Postgres => "postgres", TriggerKind::Gcp => "gcp", diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 14aeb734ed..f4659f7d82 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -13,11 +13,12 @@ default = [] private = ["windmill-worker-volumes/private", "windmill-queue/private", "windmill-common/private", "windmill-dep-map/private", "windmill-runtime-nativets?/private"] mcp = ["windmill-ai/mcp", "dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:pem", "dep:rsa", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "windmill-runtime-nativets?/enterprise", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-rustls", "dep:tokio-rustls", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pemfile", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth bigquery = ["dep:gcp_auth"] +snowflake = ["dep:pem", "dep:rsa"] # pem/rsa: key-pair auth benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"] parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] flow_testing = [] @@ -115,8 +116,11 @@ hmac.workspace = true pem = { workspace = true, optional = true } rsa = { workspace = true, optional = true } urlencoding.workspace = true -# `fs` adds flock(2) for the cross-process Python install lock (shared cache mounts) -nix = { workspace = true, features = ["fs"] } +# `user` adds geteuid(2) to verify ownership of the ansible socket-dir root +nix = { workspace = true, features = ["user"] } +# Cross-platform advisory file lock (flock on unix, LockFileEx on windows) for the +# cross-process Python install lock into shared wheel-cache dirs. +fs4 = { workspace = true } bytes.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true @@ -143,7 +147,11 @@ bollard = { workspace = true, optional = true } oracle = { workspace = true, optional = true } hudsucker = { workspace = true, optional = true } hyper-http-proxy = { workspace = true, optional = true } -hyper-tls = { workspace = true, optional = true } +hyper-rustls = { workspace = true, optional = true } +tokio-rustls = { workspace = true, optional = true } +rustls = { workspace = true, optional = true } +rustls-native-certs = { workspace = true, optional = true } +rustls-pemfile = { workspace = true, optional = true } hyper-util = { workspace = true, optional = true } rcgen = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto index 2c731e34f9..afaf066f17 100644 --- a/backend/windmill-worker/nsjail/run.ansible.config.proto +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -5,7 +5,7 @@ hostname: "ansible" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/nsjail/run.docker.config.proto b/backend/windmill-worker/nsjail/run.docker.config.proto index a2da459fbe..5cc1e4ca03 100644 --- a/backend/windmill-worker/nsjail/run.docker.config.proto +++ b/backend/windmill-worker/nsjail/run.docker.config.proto @@ -77,6 +77,13 @@ mount { is_bind: true } +mount { + dst: "/dev/shm" + fstype: "tmpfs" + rw: true + is_bind: false +} + # Host DNS config layered over the image's /etc so name resolution works on the # job's network (mandatory:false: some minimal images have no /etc files to shadow). mount { diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 53d5a6c64d..269ae1f0f8 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -5,7 +5,7 @@ hostname: "python" log_level: ERROR time_limit: {TIMEOUT} -rlimit_as: 4096 +{RLIMIT_AS} rlimit_cpu: 1000 rlimit_fsize: 1000 rlimit_nofile: 10000 diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 879f560916..8605bca374 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -26,7 +26,7 @@ use windmill_ai::{ providers::create_query_builder, query_builder::{BuildRequestArgs, ParsedResponse}, types::*, - utils::{should_use_structured_output_tool, AI_HTTP_CLIENT, AI_HTTP_HEADERS}, + utils::{pinned_ai_client_for, should_use_structured_output_tool, AI_HTTP_HEADERS}, }; use windmill_common::{ cache, @@ -984,11 +984,14 @@ pub async fn run_agent( let resource_headers = &credentials.custom_headers; + // `endpoint` derives from the user-controlled provider base_url, so pin + // DNS to the SSRF-validated address: the connect must not rebind to an + // internal IP between the check and the request (TOCTOU). + let pinned_ai_client = pinned_ai_client_for(base_url).await?; + // Helper to build HTTP request with headers let build_http_request = |body: String| { - // `endpoint` derives from the user-controlled provider base_url: use - // AI_HTTP_CLIENT, not the shared HTTP_CLIENT. See AI_HTTP_CLIENT. - let mut req = AI_HTTP_CLIENT + let mut req = pinned_ai_client .post(&endpoint) .timeout(timeout) .header("Content-Type", "application/json"); diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 292246c62c..d3e71d8fdb 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -31,13 +31,14 @@ use crate::{ bash_executor::BIN_BASH, common::{ build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, - read_and_check_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, - start_child_process, transform_json, OccupancyMetrics, + read_and_check_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, is_sandboxing_enabled, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_ANSIBLE_RLIMIT_AS_MB, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -54,6 +55,191 @@ const WINDMILL_ANSIBLE_PASSWORD_FILENAME: &str = ".windmill.ansible_vault_passwo const DELEGATE_GIT_REPO_TARGET: &str = "delegate_git_repository"; +/// Usable bytes in `sockaddr_un.sun_path` (108 minus the NUL). An ABI constant, not a +/// filesystem limit — which is why only the socket breaks while every regular file in the +/// same job dir is fine. +const AF_UNIX_PATH_LIMIT: usize = 107; + +/// Root for the per-job dir in which ansible's persistent-connection plugins +/// (`network_cli`, `httpapi`, `netconf`) bind their unix socket, named after a digest of the +/// connection. `sockaddr_un.sun_path` caps the whole socket path at [`AF_UNIX_PATH_LIMIT`], +/// which the job dir alone already exhausts, so the socket dir must stay short and cannot +/// live under `ANSIBLE_HOME` (which Windmill pins into the job dir). +/// +/// Fixed, and directly under `/tmp`, for two reasons that are easy to undo by accident: +/// `/tmp`'s sticky bit is what stops another uid renaming our root away, the one property +/// [`prepare_socket_root`] needs from a parent; and every component of a fixed path is one +/// nobody can point elsewhere, so trusting the root does not mean trusting an ancestor +/// chain. Notably NOT under `WINDMILL_DIR`: the shipped image chmods that tree to a +/// non-sticky 0777 so any UID can write it (`Dockerfile`, "Make directories +/// world-accessible for any UID"), which is exactly the parent an attacker can swap entries +/// in. +const PERSISTENT_CONTROL_PATH_ROOT: &str = "/tmp/wm-pc"; + +/// Ansible's env var for `[persistent_connection] control_path_dir`. +const ANSIBLE_CONTROL_PATH_DIR_ENV: &str = "ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"; + +/// The budget this whole change exists to protect: root + `/` + a 32-char job uuid + `/` + +/// a socket name, allowing a full 40-char sha1 (ansible truncates it far shorter today, but +/// a custom control path may not). +const _: () = assert!(PERSISTENT_CONTROL_PATH_ROOT.len() + 1 + 32 + 1 + 40 <= AF_UNIX_PATH_LIMIT); + +/// Cleared when the root cannot be trusted (see [`prepare_socket_root`]), which makes jobs +/// stop naming it and fall back to ansible's own `{ANSIBLE_HOME}/pc` default — inside the +/// job dir, so worker-owned. Network playbooks then fail on the path length as they did +/// before this dir existed, which beats handing an attacker the socket a device session +/// runs over. Defaults to trusted: the check runs at worker start, before any job. +static SOCKET_ROOT_TRUSTED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Socket dir for `job_id`, or `None` when the root is untrusted. Per-job on purpose: +/// socket names hash host+credentials, so concurrent jobs sharing a dir would reuse each +/// other's connection daemon. +fn persistent_control_path_dir(job_id: &Uuid) -> Option { + SOCKET_ROOT_TRUSTED + .load(std::sync::atomic::Ordering::Relaxed) + .then(|| format!("{PERSISTENT_CONTROL_PATH_ROOT}/{}", job_id.simple())) +} + +/// Whether `name` is one this module could have created, i.e. `Uuid::simple` (32 hex, no +/// hyphens). Belt to the root check's braces: nothing else should ever be in there. +#[cfg(unix)] +fn is_persistent_control_path_dir_name(name: &str) -> bool { + name.len() == 32 && Uuid::try_parse(name).is_ok() +} + +/// Removes the job's socket dir on the way out. It lives outside `job_dir`, so the +/// worker's job-dir sweep does not cover it. +struct PersistentControlPathGuard(String); + +impl Drop for PersistentControlPathGuard { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +/// Claim the socket-dir root at worker start, then reap dirs left behind by workers that +/// died before their guard could run. +#[cfg(unix)] +pub async fn prepare_persistent_control_path_root() { + // A play holds its socket dir for as long as it runs, touching the mtime only when + // connections open, so anything younger than the longest permitted job may still be + // live — including on another worker sharing this host. + let stale_after = std::time::Duration::from_secs( + windmill_common::worker::MAX_TIMEOUT.saturating_add(24 * 60 * 60), + ); + prepare_socket_root(PERSISTENT_CONTROL_PATH_ROOT, stale_after).await +} + +/// Reject a root that another local user could control, and mark it untrusted so jobs stop +/// naming it. Returns without sweeping in that case. +/// +/// SECURITY: the root sits in a world-writable `/tmp`, so a local user who wins the race to +/// create it owns the parent of every job's socket dir — +/// enough to hand ansible a socket of their choosing (a device session, credentials and +/// all, runs over it), or to swap in a symlink and redirect the sweep's path-based +/// `remove_dir_all` onto a target of their choosing, as the worker's uid. Three things must +/// hold: the root is a real directory (`symlink_metadata` reports the link's own type +/// without following it, so `is_dir()` cannot be satisfied by a symlink), we own it and +/// nobody else can write it, and its parent cannot be used to replace it — which needs the +/// parent either not writable by others, or sticky, since the sticky bit is exactly what +/// stops a non-owner renaming an entry out of a shared dir. The root is validated after the +/// create attempt, never before: anything else races whoever creates it first. +#[cfg(unix)] +async fn prepare_socket_root(root: &str, stale_after: std::time::Duration) { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let untrusted = |reason: String| { + tracing::error!( + "Refusing to use the ansible persistent-connection socket root at {root}: {reason}. \ + Ansible network playbooks on this host will keep failing with `AF_UNIX path too \ + long` until this is resolved." + ); + SOCKET_ROOT_TRUSTED.store(false, std::sync::atomic::Ordering::Relaxed); + }; + + if let Some(parent) = std::path::Path::new(root).parent() { + // Resolved, not `symlink_metadata`: what matters is the mode of the directory the + // entries actually live in, and a symlinked parent is normal (macOS `/tmp`). + match tokio::fs::metadata(parent).await { + Ok(meta) => { + let mode = meta.permissions().mode(); + if mode & 0o022 != 0 && mode & 0o1000 == 0 { + return untrusted(format!( + "its parent {} is writable by other users and not sticky (mode={:o}), \ + so they could replace the root", + parent.display(), + mode & 0o7777 + )); + } + } + Err(e) => return untrusted(format!("cannot stat its parent: {e}")), + } + } + + // Non-recursive on purpose: `recursive` reports success for a path that already + // exists, which under a sticky parent (where others may still *create* the + // not-yet-existing `pc`, only not rename ours away) would hand us whatever another uid + // raced into place. Create-or-EEXIST, then validate whatever is actually there. + match tokio::fs::DirBuilder::new().mode(0o700).create(root).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(e) => return untrusted(format!("it could not be created: {e}")), + } + + match tokio::fs::symlink_metadata(root).await { + Ok(meta) if meta.is_dir() => { + let mode = meta.permissions().mode(); + // Trusted means usable as well as safe: without owner rwx ansible cannot create + // its per-job dir, and a trusted-but-unusable root would hand every network + // playbook a permission error instead of the working fallback. + if meta.uid() != nix::unistd::Uid::effective().as_raw() + || mode & 0o022 != 0 + || mode & 0o700 != 0o700 + { + return untrusted(format!( + "it is not owned by this worker, is writable by others, or is not \ + writable by us (uid={}, mode={:o})", + meta.uid(), + mode & 0o7777 + )); + } + } + Ok(_) => { + return untrusted( + "it is not a directory (possibly a symlink planted by another local user)" + .to_string(), + ) + } + Err(e) => return untrusted(format!("it could not be stat'd: {e}")), + } + + let Ok(mut entries) = tokio::fs::read_dir(root).await else { + return; + }; + while let Ok(Some(entry)) = entries.next_entry().await { + // Only reap what we could have created. Nothing else should ever be in a root we + // made 0700 ourselves, but this is a recursive delete running as the worker's uid: + // cheap to bound by name, expensive to get wrong. + if !entry + .file_name() + .to_str() + .is_some_and(is_persistent_control_path_dir_name) + { + continue; + } + // `DirEntry::metadata` does not traverse symlinks, so a planted link is never + // followed here either. + let stale = match entry.metadata().await.and_then(|m| m.modified()) { + Ok(modified) => modified.elapsed().is_ok_and(|e| e > stale_after), + Err(_) => false, + }; + if stale { + let _ = tokio::fs::remove_dir_all(entry.path()).await; + } + } +} + lazy_static::lazy_static! { static ref TEMPLATE_RE: regex::Regex = regex::Regex::new(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}").unwrap(); } @@ -902,6 +1088,7 @@ pub fn create_ansible_cfg( reqs: Option<&AnsibleRequirements>, job_dir: &str, vault_password_file_exists: bool, + job_id: &Uuid, ) -> error::Result<()> { let mut passwords_cfg = String::new(); if vault_password_file_exists { @@ -921,6 +1108,9 @@ pub fn create_ansible_cfg( passwords_cfg.push_str(&format!("vault_identity_list = {password_files}\n")); } } + let persistent_cfg = persistent_control_path_dir(job_id) + .map(|dir| format!("[persistent_connection]\ncontrol_path_dir = {dir}\n")) + .unwrap_or_default(); let ansible_cfg_content = format!( r#" [defaults] @@ -930,7 +1120,7 @@ home={job_dir}/.ansible local_tmp={job_dir}/.ansible/tmp remote_tmp={job_dir}/.ansible/tmp {passwords_cfg} -"# +{persistent_cfg}"# ); write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; @@ -938,6 +1128,15 @@ remote_tmp={job_dir}/.ansible/tmp Ok(()) } +/// The section a header line opens, if it is one. Mirrors configparser's `SECTCRE` +/// (`\[(?P
.+)\]`, matched not fullmatched, `.+` greedy): the name runs to the +/// *last* `]`, and anything after it — an inline comment, say — is ignored. +fn parse_ansible_cfg_section_header(trimmed: &str) -> Option<&str> { + let rest = trimmed.strip_prefix('[')?; + let end = rest.rfind(']')?; + Some(rest[..end].trim()) +} + /// Read a colon-separated path list (e.g. `roles_path`, `collections_path`) from /// the `[defaults]` section of an ansible.cfg. Returns the raw entries as written, /// unresolved. Deliberately minimal: no inline-comment or continuation handling, @@ -948,10 +1147,8 @@ fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> let mut in_defaults = false; for line in content.lines() { let trimmed = line.trim(); - if trimmed.starts_with('[') && trimmed.ends_with(']') { - in_defaults = trimmed[1..trimmed.len() - 1] - .trim() - .eq_ignore_ascii_case("defaults"); + if let Some(section) = parse_ansible_cfg_section_header(trimmed) { + in_defaults = section.eq_ignore_ascii_case("defaults"); continue; } if !in_defaults || trimmed.starts_with('#') || trimmed.starts_with(';') { @@ -977,6 +1174,29 @@ fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> None } +/// Whether `section` declares `key` in an ansible.cfg. Same deliberately minimal +/// parsing as [`parse_ansible_cfg_path_list`], for a scalar key in a named section. +fn ansible_cfg_declares(content: &str, section: &str, key: &str) -> bool { + let mut in_section = false; + for line in content.lines() { + let trimmed = line.trim(); + if let Some(header) = parse_ansible_cfg_section_header(trimmed) { + in_section = header.eq_ignore_ascii_case(section); + continue; + } + if !in_section || trimmed.starts_with('#') || trimmed.starts_with(';') { + continue; + } + let sep = trimmed.find('=').into_iter().chain(trimmed.find(':')).min(); + if let Some(sep) = sep { + if trimmed[..sep].trim().eq_ignore_ascii_case(key) { + return true; + } + } + } + false +} + /// Prepend Windmill's dependency install dir to the repo cfg's declared path list. /// Relative entries from the repo cfg are resolved against `cfg_dir` to match how /// ansible resolves them relative to the config file's own directory. @@ -1005,6 +1225,8 @@ async fn build_ansible_cfg_override_envs( job_dir: &str, vault_password_file_exists: bool, reqs: Option<&AnsibleRequirements>, + job_id: &Uuid, + job_envs: &HashMap, ) -> error::Result> { let mut envs = vec![ ("ANSIBLE_CONFIG".to_string(), cfg_path.to_string()), @@ -1052,6 +1274,18 @@ async fn build_ansible_cfg_override_envs( )) })?; + // Persistent-connection socket dir: only a default. Unlike ANSIBLE_HOME this value is + // not runtime-bound, so a repo that picks its own dir keeps it — and so does a job that + // sets the env var itself, which these overrides are applied after and would otherwise + // silently outrank. + if !ansible_cfg_declares(&cfg_content, "persistent_connection", "control_path_dir") + && !job_envs.contains_key(ANSIBLE_CONTROL_PATH_DIR_ENV) + { + if let Some(dir) = persistent_control_path_dir(job_id) { + envs.push((ANSIBLE_CONTROL_PATH_DIR_ENV.to_string(), dir)); + } + } + envs.push(( "ANSIBLE_ROLES_PATH".to_string(), resolve_and_prepend_path( @@ -1605,7 +1839,8 @@ pub async fn handle_ansible_job( None => false, }; - create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?; + create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists, &job.id)?; + let _control_path_guard = persistent_control_path_dir(&job.id).map(PersistentControlPathGuard); // When the run delegates to a git repo that ships its own ansible.cfg, that // file becomes the effective config (ansible loads exactly one config file and @@ -1627,6 +1862,8 @@ pub async fn handle_ansible_job( job_dir, vault_password_file_exists, reqs.as_ref(), + &job.id, + &envs, ) .await? } @@ -1659,6 +1896,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_ANSIBLE_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{JOB_DIR}", job_dir) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) @@ -1936,6 +2177,10 @@ async fn get_resource_or_variable_content( mod tests { use super::*; + fn no_job_envs() -> HashMap { + HashMap::new() + } + fn args_from_json(v: serde_json::Value) -> HashMap> { let serde_json::Value::Object(map) = v else { panic!("expected object"); @@ -2022,12 +2267,40 @@ mod tests { vault_id: vec!["dev@vault_pass.txt".to_string()], ..Default::default() }; - create_ansible_cfg(Some(&reqs), job_dir, false).unwrap(); + create_ansible_cfg(Some(&reqs), job_dir, false, &Uuid::new_v4()).unwrap(); let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); assert!(cfg.contains("vault_identity_list = dev@vault_pass.txt")); assert!(!cfg.contains("library")); } + /// The socket ansible binds under `control_path_dir` must fit `sun_path` (107 + /// usable bytes), which the job dir alone blows past — hence a short dir outside + /// `ANSIBLE_HOME`. + #[test] + fn test_create_ansible_cfg_control_path_dir_fits_af_unix_limit() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let job_id = Uuid::new_v4(); + let flag = TrustFlag::lock(); + flag.set(true); + create_ansible_cfg(None, job_dir, false, &job_id).unwrap(); + + let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); + let control_path_dir = persistent_control_path_dir(&job_id).unwrap(); + assert!(cfg.contains("[persistent_connection]")); + assert!(cfg.contains(&format!("control_path_dir = {control_path_dir}"))); + // The whole point: the socket dir must escape the job dir, whose length is what + // blows the budget. + assert!(!control_path_dir.starts_with(job_dir)); + + // dir + `/` + socket name, budgeted at a full 40-char sha1 (ansible truncates + // it far shorter today, but a custom control path may not). + assert!( + control_path_dir.len() + 1 + 40 <= AF_UNIX_PATH_LIMIT, + "socket path would exceed sun_path: {control_path_dir}" + ); + } + #[test] fn test_create_ansible_cfg_rejects_vault_id_injection() { let dir = tempfile::tempdir().unwrap(); @@ -2037,7 +2310,7 @@ mod tests { ..Default::default() }; // Defense-in-depth boundary: a poisoned entry must error before any config is written. - assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err()); + assert!(create_ansible_cfg(Some(&reqs), job_dir, false, &Uuid::new_v4()).is_err()); assert!(!dir.path().join("ansible.cfg").exists()); } @@ -2156,7 +2429,7 @@ collections_path : a/col:b/col std::fs::write(repo.join("play.yml"), play).unwrap(); // Windmill's own generated cfg (the negative-control config that exists today). - create_ansible_cfg(None, job_dir, false).unwrap(); + create_ansible_cfg(None, job_dir, false, &Uuid::new_v4()).unwrap(); let playbook = format!("{DELEGATE_GIT_REPO_TARGET}/play.yml"); let run = |envs: Vec<(String, String)>| { @@ -2180,10 +2453,16 @@ collections_path : a/col:b/col // With the override: ANSIBLE_CONFIG points at the repo cfg and roles_path // is honored, so the role runs. - let envs = - build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) - .await - .unwrap(); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); let after = run(envs); let stdout = String::from_utf8_lossy(&after.stdout); assert!( @@ -2208,15 +2487,30 @@ collections_path : a/col:b/col vault_id: vec!["dev@vault_pass.txt".to_string()], ..Default::default() }; - let envs = build_ansible_cfg_override_envs(cfg_path, job_dir, true, Some(&reqs)) - .await - .unwrap(); + let job_id = Uuid::new_v4(); + let flag = TrustFlag::lock(); + flag.set(true); + let envs = build_ansible_cfg_override_envs( + cfg_path, + job_dir, + true, + Some(&reqs), + &job_id, + &no_job_envs(), + ) + .await + .unwrap(); let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); assert_eq!( map.get("ANSIBLE_CONFIG").map(|s| s.as_str()), Some(cfg_path) ); + // The repo cfg declares no control_path_dir, so Windmill's short default applies. + assert_eq!( + map.get("ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"), + persistent_control_path_dir(&job_id).as_ref() + ); assert_eq!( map.get("ANSIBLE_HOME"), Some(&format!("{job_dir}/.ansible")) @@ -2256,14 +2550,424 @@ collections_path : a/col:b/col // are not silently dropped when the env override replaces the cfg value. std::fs::write(&cfg_path, "[defaults]\ncollections_paths = my_cols\n").unwrap(); - let envs = - build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) - .await - .unwrap(); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); assert_eq!( map.get("ANSIBLE_COLLECTIONS_PATH"), Some(&format!("{job_dir}:{}/my_cols", repo_dir.to_str().unwrap())) ); } + + /// These overrides are applied after the job's own env, so a default that ignores what + /// the job set would silently outrank it. Not runtime-bound, so the job wins. + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_keeps_job_env_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join(DELEGATE_GIT_REPO_TARGET); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + // Cfg is silent on control_path_dir; the job env is not. + std::fs::write(&cfg_path, "[defaults]\nroles_path = my_roles\n").unwrap(); + + let job_envs = HashMap::from([( + ANSIBLE_CONTROL_PATH_DIR_ENV.to_string(), + "/tmp/job-picked".to_string(), + )]); + + let flag = TrustFlag::lock(); + flag.set(true); + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &job_envs, + ) + .await + .unwrap(); + + assert!( + !envs.iter().any(|(k, _)| k == ANSIBLE_CONTROL_PATH_DIR_ENV), + "must not override a control_path_dir the job set itself" + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_keeps_user_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join(DELEGATE_GIT_REPO_TARGET); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + std::fs::write( + &cfg_path, + "[defaults]\nroles_path = my_roles\n\n[persistent_connection]\ncontrol_path_dir = /tmp/my_pc\n", + ) + .unwrap(); + + let envs = build_ansible_cfg_override_envs( + cfg_path.to_str().unwrap(), + job_dir, + false, + None, + &Uuid::new_v4(), + &no_job_envs(), + ) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + assert_eq!(map.get("ANSIBLE_PERSISTENT_CONTROL_PATH_DIR"), None); + } + + /// `SOCKET_ROOT_TRUSTED` is process-global and cargo runs tests in parallel: hold this + /// while reading or flipping it, and the default is restored on the way out. + struct TrustFlag(#[allow(dead_code)] std::sync::MutexGuard<'static, ()>); + + impl TrustFlag { + fn lock() -> Self { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + Self(LOCK.lock().unwrap_or_else(|e| e.into_inner())) + } + fn set(&self, trusted: bool) { + SOCKET_ROOT_TRUSTED.store(trusted, std::sync::atomic::Ordering::Relaxed); + } + fn get(&self) -> bool { + SOCKET_ROOT_TRUSTED.load(std::sync::atomic::Ordering::Relaxed) + } + } + + impl Drop for TrustFlag { + fn drop(&mut self) { + SOCKET_ROOT_TRUSTED.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + + #[cfg(unix)] + fn backdate(path: &std::path::Path, age: std::time::Duration) { + let times = std::fs::FileTimes::new().set_modified(std::time::SystemTime::now() - age); + std::fs::File::open(path).unwrap().set_times(times).unwrap(); + } + + /// The sweep only reaps what no live job can own: a play may hold its socket dir for + /// the whole of MAX_TIMEOUT without touching the mtime again. And it only ever touches + /// names it could have created itself. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_sweeps_only_stale_dirs() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + + let stale = root.join(Uuid::new_v4().simple().to_string()); + let live = root.join(Uuid::new_v4().simple().to_string()); + let foreign = root.join("someone-elses-data"); + for p in [&stale, &live, &foreign] { + std::fs::create_dir(p).unwrap(); + } + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + backdate(&live, std::time::Duration::from_secs(12 * 60 * 60)); + backdate(&foreign, std::time::Duration::from_secs(48 * 60 * 60)); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(!stale.exists(), "dir older than the cutoff must be reaped"); + assert!(live.exists(), "a dir a live job may still own must be kept"); + assert!( + foreign.exists(), + "a stale dir we never created must be left alone" + ); + } + + #[cfg(unix)] + #[test] + fn test_is_persistent_control_path_dir_name() { + assert!(is_persistent_control_path_dir_name( + &Uuid::new_v4().simple().to_string() + )); + // Hyphenated form is not what we create, so it is not ours to delete. + assert!(!is_persistent_control_path_dir_name( + &Uuid::new_v4().to_string() + )); + assert!(!is_persistent_control_path_dir_name("someone-elses-data")); + assert!(!is_persistent_control_path_dir_name("")); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_creates_root_private() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + let meta = std::fs::metadata(&root).unwrap(); + assert!(meta.is_dir()); + // Owning the root 0700 is what stops another local user replacing it later. + assert_eq!(meta.permissions().mode() & 0o777, 0o700); + assert!(flag.get(), "a root we created ourselves is trusted"); + } + + /// The root must be validated *after* the create attempt, not before: under a sticky + /// parent another uid may still win the race to create the not-yet-existing `pc` + /// (sticky stops them renaming ours away, not creating it first), and a create that + /// tolerates `AlreadyExists` would otherwise hand us their directory unchecked. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_validates_raced_creation() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)).unwrap(); + + // Stand in for the racer's dir: present before we look, and not exclusively ours. + let root = parent.join("pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + assert!( + !flag.get(), + "a root raced into place under a sticky parent must not be trusted" + ); + } + + /// Safe but unusable is still not trusted: ansible cannot create its per-job dir under + /// a root we cannot write, and naming it anyway would swap the working fallback for a + /// permission error on every network playbook. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_unwritable_root() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o500)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root(root.to_str().unwrap(), std::time::Duration::from_secs(1)).await; + + assert!(!flag.get(), "a root we cannot write must not be trusted"); + // Let the tempdir clean itself up. + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + + /// A root we do not exclusively own may have been pre-planted by another local user, + /// who then controls the parent of every job's socket dir — and could swap a symlink + /// in after this check, redirecting the sweep's path-based `remove_dir_all`. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_world_writable_root() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("wm-pc"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); + + // UUID-named, so survival proves the trust check stopped the sweep rather than the + // name filter. + let stale = root.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir(&stale).unwrap(); + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!( + stale.exists(), + "must not sweep a root that others can write to" + ); + } + + /// The root must hang off `/tmp`, whose sticky bit is what protects it. The trap this + /// guards: the shipped image chmods the whole `WINDMILL_DIR` tree to a non-sticky 0777 + /// so any UID can write it, so parenting the root there would make it untrusted and + /// silently disable this fix in the standard image while every local test still passed. + #[test] + fn test_control_path_root_hangs_off_tmp() { + assert_eq!( + std::path::Path::new(PERSISTENT_CONTROL_PATH_ROOT).parent(), + Some(std::path::Path::new("/tmp")) + ); + } + + /// A parent that others can write (and that is not sticky) lets them rename the root + /// away and drop a symlink in its place after the checks — so the root cannot be + /// trusted no matter how it currently looks. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_writable_parent() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + let root = parent.join("pc"); + std::fs::create_dir_all(&root).unwrap(); + // UUID-named, so survival proves the trust check stopped the sweep rather than the + // name filter. + let stale = root.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir(&stale).unwrap(); + backdate(&stale, std::time::Duration::from_secs(48 * 60 * 60)); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(stale.exists(), "must not sweep under a replaceable parent"); + assert!( + !flag.get(), + "an untrusted root must be marked so jobs stop naming it" + ); + } + + /// A sticky parent (like /tmp itself) is fine: the sticky bit is what stops a + /// non-owner renaming our root out of it. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_accepts_sticky_world_writable_parent() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("windmill"); + let root = parent.join("pc"); + std::fs::create_dir_all(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o1777)).unwrap(); + + let flag = TrustFlag::lock(); + flag.set(true); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!(root.is_dir(), "root must be created under a sticky parent"); + assert!(flag.get()); + } + + /// Fail closed: when the root is untrusted the cfg must not name it, so ansible falls + /// back to its own `{ANSIBLE_HOME}/pc` default inside the worker-owned job dir. + #[test] + fn test_create_ansible_cfg_omits_untrusted_control_path_dir() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + + let flag = TrustFlag::lock(); + flag.set(false); + create_ansible_cfg(None, job_dir, false, &Uuid::new_v4()).unwrap(); + + let cfg = std::fs::read_to_string(dir.path().join("ansible.cfg")).unwrap(); + assert!(!cfg.contains("control_path_dir")); + assert!(!cfg.contains("[persistent_connection]")); + } + + /// A symlinked root must never be swept: `remove_dir_all` through it would delete + /// whatever the link points at, as the worker's uid. + #[cfg(unix)] + #[tokio::test] + async fn test_prepare_socket_root_refuses_symlinked_root() { + let dir = tempfile::tempdir().unwrap(); + let victim = dir.path().join("victim"); + // UUID-named, so survival proves the symlink was not followed rather than the name + // filter sparing it. + let victim_child = victim.join(Uuid::new_v4().simple().to_string()); + std::fs::create_dir_all(&victim_child).unwrap(); + backdate(&victim_child, std::time::Duration::from_secs(48 * 60 * 60)); + + let root = dir.path().join("wm-pc"); + std::os::unix::fs::symlink(&victim, &root).unwrap(); + + let _flag = TrustFlag::lock(); + prepare_socket_root( + root.to_str().unwrap(), + std::time::Duration::from_secs(24 * 60 * 60), + ) + .await; + + assert!( + victim_child.exists(), + "sweep must not follow a symlinked root" + ); + } + + /// configparser matches `\[(?P
.+)\]` without anchoring the end of the line, so + /// a header with anything trailing it is still that section — and missing it here + /// would silently override the user's own control_path_dir. + #[test] + fn test_ansible_cfg_section_header_with_trailing_text() { + assert_eq!( + parse_ansible_cfg_section_header("[persistent_connection] ; note"), + Some("persistent_connection") + ); + assert_eq!(parse_ansible_cfg_section_header("not a header"), None); + // Greedy `.+` runs to the last `]`. + assert_eq!(parse_ansible_cfg_section_header("[a]b]"), Some("a]b")); + + assert!(ansible_cfg_declares( + "[persistent_connection] ; note\ncontrol_path_dir = /tmp/mine\n", + "persistent_connection", + "control_path_dir" + )); + assert_eq!( + parse_ansible_cfg_path_list("[defaults] # note\nroles_path = my_roles\n", "roles_path"), + Some(vec!["my_roles".to_string()]) + ); + } + + #[test] + fn test_ansible_cfg_declares_scoped_to_section() { + let cfg = "\ +[defaults] +control_path_dir = /wrong/section + +[persistent_connection] +# control_path_dir = /commented +connect_timeout = 30 +"; + assert!(!ansible_cfg_declares( + cfg, + "persistent_connection", + "control_path_dir" + )); + assert!(ansible_cfg_declares( + cfg, + "persistent_connection", + "connect_timeout" + )); + } } diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 8a80de25ef..b383b22966 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -412,15 +412,20 @@ pub async fn do_bigquery( let s3_obj: windmill_types::s3::S3Object = serde_json::from_value(raw).map_err(|e| { Error::ExecutionErr(format!("Invalid S3Object for arg `{}`: {e}", arg.name)) })?; - let json_text = - crate::sql_s3_input::fetch_s3object_as_json_text(client, &job.workspace_id, &s3_obj) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Failed to fetch S3 object for arg `{}`: {e}", - arg.name - )) - })?; + let json_text = crate::sql_s3_input::fetch_s3object_as_json_text( + client, + conn, + job.id, + &job.workspace_id, + &s3_obj, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to fetch S3 object for arg `{}`: {e}", + arg.name + )) + })?; bigquery_args.insert(arg.name.clone(), Value::String(json_text)); arg.otyp = Some("string".to_string()); } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f6b093f766..0f9f73ad1d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3125,15 +3125,12 @@ pub async fn handle_wac_v2_output( } } - // Generate resume URLs for the inline approval buttons. - // Use a hash of the step key as resume_id so each waitForApproval() - // in the same workflow gets a unique resume_job record. - let resume_id: u32 = { - use std::hash::{Hash, Hasher}; - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - key.hash(&mut hasher); - (hasher.finish() & 0xFFFF_FFFF) as u32 - }; + // Generate resume URLs for the inline approval buttons. The resume_id + // is derived from the step key so each waitForApproval() in the same + // workflow gets a unique resume_job record, and so URLs the workflow + // minted for this step ahead of time (getApprovalUrls) address the + // same one. + let resume_id: u32 = windmill_common::wac::approval_resume_id(&key); // Generate stateless approval token using shared utility let approval_token = windmill_common::variables::generate_approval_token(&job.workspace_id, job.id, db) diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index a88ba67bb7..a406f3610c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1081,7 +1081,9 @@ pub async fn resolve_job_timeout( *MAX_TIMEOUT_DURATION }; - match custom_timeout_secs { + // A `custom_timeout_secs <= 0` is not a 0-second limit but "unset": fall through to the + // default/global-max timeout instead of killing the job immediately. + match windmill_common::runnable_settings::none_if_non_positive(custom_timeout_secs) { Some(timeout_secs) if Duration::from_secs(timeout_secs as u64) < global_max_timeout_duration => { @@ -1124,6 +1126,45 @@ pub async fn resolve_nsjail_timeout( (duration.as_secs() + 15).to_string() } +/// Render the `rlimit_as` line for an nsjail run config, honoring a per-language +/// env-var override. +/// +/// nsjail caps a jailed job's virtual address space at `rlimit_as` MiB. JIT +/// runtimes (Bun/JavaScriptCore, the JVM) reserve large virtual ranges up front, +/// so a subprocess spawned from a jailed Python/Ansible job can crash against this +/// cap even when its physical memory use is modest. Lifting it lets operators run +/// such workloads on a dedicated worker pool (set the env var only there) without +/// giving up the mount/PID/user-namespace isolation that provides the real +/// security boundary. Only the address-space limit is affected; the other rlimits +/// (cpu/fsize/nofile) in the proto are untouched. +/// +/// `env_override` is the raw value of the language's `NSJAIL_*_RLIMIT_AS_MB` env var: +/// - unset/empty -> historical default (`rlimit_as: {default_mb}`) +/// - `unlimited`/`none`/`inf`/`0` -> `rlimit_as_type: INF` (address space uncapped) +/// - a positive integer (MiB) -> `rlimit_as: {n}` +pub fn render_nsjail_rlimit_as(env_override: Option<&str>, default_mb: u32) -> String { + match env_override.map(str::trim) { + None | Some("") => format!("rlimit_as: {default_mb}"), + Some(v) + if v.eq_ignore_ascii_case("unlimited") + || v.eq_ignore_ascii_case("none") + || v.eq_ignore_ascii_case("inf") + || v == "0" => + { + "rlimit_as_type: INF".to_string() + } + Some(v) => match v.parse::() { + Ok(mb) => format!("rlimit_as: {mb}"), + Err(_) => { + tracing::warn!( + "Invalid nsjail rlimit_as override {v:?}, using default {default_mb}MiB" + ); + format!("rlimit_as: {default_mb}") + } + }, + } +} + /// Default size (in bytes) of the `/tmp` tmpfs mount inside nsjail sandboxes, /// used when the `nsjail_tmpfs_size_mb` instance setting is unset. pub const DEFAULT_NSJAIL_TMPFS_SIZE_BYTES: u64 = 800_000_000; @@ -1233,6 +1274,49 @@ pub(crate) async fn resolve_nsjail_tmp_mount_block(job_dir: &str) -> String { bind_mount_block(&jail_tmp) } +#[cfg(test)] +mod nsjail_rlimit_as_tests { + use super::render_nsjail_rlimit_as; + + #[test] + fn unset_uses_default() { + assert_eq!(render_nsjail_rlimit_as(None, 4096), "rlimit_as: 4096"); + assert_eq!(render_nsjail_rlimit_as(Some(" "), 4096), "rlimit_as: 4096"); + } + + #[test] + fn numeric_override_is_used() { + assert_eq!( + render_nsjail_rlimit_as(Some("16384"), 4096), + "rlimit_as: 16384" + ); + assert_eq!( + render_nsjail_rlimit_as(Some(" 8192 "), 4096), + "rlimit_as: 8192" + ); + } + + #[test] + fn unlimited_keywords_emit_inf() { + for v in ["unlimited", "UNLIMITED", "none", "inf", "0"] { + assert_eq!( + render_nsjail_rlimit_as(Some(v), 4096), + "rlimit_as_type: INF", + "value {v:?}" + ); + } + } + + #[test] + fn invalid_falls_back_to_default() { + assert_eq!( + render_nsjail_rlimit_as(Some("abc"), 4096), + "rlimit_as: 4096" + ); + assert_eq!(render_nsjail_rlimit_as(Some("-1"), 4096), "rlimit_as: 4096"); + } +} + #[cfg(test)] mod nsjail_tmp_mount_tests { use super::*; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e4e24d6653..e2807547e7 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2571,15 +2571,57 @@ async fn transform_attach_datatable( } Connection::Sql(db) => get_datatable_resource_from_db_unchecked(db, w_id, name).await?, }; - let db_type = "postgres"; if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { hidden_passwords.lock().unwrap().push(pwd.to_string()); } - Ok(Some( - db_resource_to_attach_statements(db_resource, alias_name, db_type, None).await?, - )) + Ok(Some(pg_secret_attach_statements(db_resource, alias_name)?)) +} + +// Secret names must be plain identifiers; the hash keeps two aliases distinct even +// when sanitizing maps them to the same string. +fn datatable_secret_name(alias: &str) -> String { + use sha2::{Digest, Sha256}; + let sanitized: String = alias + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + let hash = &Sha256::digest(alias.as_bytes())[..4]; + format!( + "__wm_datatable_{sanitized}_{:08x}", + u32::from_be_bytes([hash[0], hash[1], hash[2], hash[3]]) + ) +} + +/// ATTACH a datatable's postgres database through a DuckDB TEMPORARY SECRET holding +/// the connection parameters; only sslmode rides in the ATTACH string. +fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result> { + let res: PgDatabase = serde_json::from_value(db_resource)?; + // Escape single quotes: each field is embedded in a single-quoted DuckDB literal, + // so an unescaped quote would break out of the CREATE SECRET statement. + let esc = |s: &str| s.replace('\'', "''"); + // The postgres secret type has no sslmode parameter, so it goes in the ATTACH + // string; only the libpq values PgDatabase::to_uri collapses to are forwarded. + let sslmode = match res.sslmode.as_deref() { + Some("disable") => "disable", + Some("require") | Some("verify-ca") | Some("verify-full") => "require", + _ => "prefer", + }; + let secret_name = datatable_secret_name(alias_name); + Ok(vec![ + "INSTALL postgres;".to_string(), + "LOAD postgres;".to_string(), + format!( + "CREATE OR REPLACE TEMPORARY SECRET {secret_name} (TYPE postgres, HOST '{}', PORT {}, DATABASE '{}', USER '{}', PASSWORD '{}');", + esc(&res.host), + res.port.unwrap_or(5432), + esc(&res.dbname), + esc(res.user.as_deref().unwrap_or("postgres")), + esc(res.password.as_deref().unwrap_or("")), + ), + format!("ATTACH 'sslmode={sslmode}' AS {alias_name} (TYPE postgres, SECRET {secret_name});"), + ]) } async fn transform_s3_uris(query: &str) -> Result { @@ -3716,6 +3758,63 @@ mod tests { assert!(result.contains("sslmode=prefer")); } + #[test] + fn test_pg_secret_attach_statements() { + let db_resource = json!({ + "host": "localhost", + "port": 5433, + "user": "custom_instance_user", + "password": "it's-secret", + "dbname": "wm_datatables", + "sslmode": "require" + }); + let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + assert_eq!(stmts[0], "INSTALL postgres;"); + assert_eq!(stmts[1], "LOAD postgres;"); + let secret_name = datatable_secret_name("dt"); + assert_eq!( + stmts[2], + format!( + "CREATE OR REPLACE TEMPORARY SECRET {secret_name} (TYPE postgres, HOST 'localhost', PORT 5433, DATABASE 'wm_datatables', USER 'custom_instance_user', PASSWORD 'it''s-secret');" + ) + ); + assert_eq!( + stmts[3], + format!("ATTACH 'sslmode=require' AS dt (TYPE postgres, SECRET {secret_name});") + ); + } + + #[test] + fn test_pg_secret_attach_statements_sslmode_whitelist() { + for (input, expected) in [ + (Some("allow"), "prefer"), + (Some("verify-full"), "require"), + (Some("disable"), "disable"), + (Some("unknown-value"), "prefer"), + (None, "prefer"), + ] { + let mut db_resource = json!({ "host": "h", "dbname": "d" }); + if let Some(s) = input { + db_resource["sslmode"] = json!(s); + } + let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + assert!( + stmts[3].starts_with(&format!("ATTACH 'sslmode={expected}'")), + "sslmode {input:?} → {}", + stmts[3] + ); + } + } + + #[test] + fn test_datatable_secret_name_sanitizes_and_disambiguates() { + let a = datatable_secret_name("a.b"); + let b = datatable_secret_name("a_b"); + assert!(a.starts_with("__wm_datatable_a_b_")); + assert_ne!(a, b); + assert!(a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')); + } + #[test] fn test_format_attach_db_conn_str_bigquery() { let db_resource = json!({ diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 9fa0b514c4..fe2fffb93e 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -1,8 +1,8 @@ -#[cfg(all(feature = "enterprise", feature = "bigquery"))] +#[cfg(feature = "bigquery")] mod bigquery_executor; #[cfg(all(feature = "enterprise", feature = "mssql"))] mod mssql_executor; -#[cfg(feature = "enterprise")] +#[cfg(feature = "snowflake")] mod snowflake_executor; mod agent_workers; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index f3df3ea3b4..a5b4a06509 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -237,14 +237,15 @@ pub async fn do_mssql( let s3_obj: S3Object = serde_json::from_value(raw).map_err(|e| { Error::ExecutionErr(format!("Invalid S3Object for arg `{}`: {e}", arg.name)) })?; - let json_text = fetch_s3object_as_json_text(authed_client, &job.workspace_id, &s3_obj) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Failed to fetch S3 object for arg `{}`: {e}", - arg.name - )) - })?; + let json_text = + fetch_s3object_as_json_text(authed_client, conn, job.id, &job.workspace_id, &s3_obj) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to fetch S3 object for arg `{}`: {e}", + arg.name + )) + })?; mssql_args.insert(arg.name.clone(), Value::String(json_text)); } diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index b7234b7912..dcec77e298 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -242,15 +242,20 @@ pub async fn do_mysql( let s3_obj: windmill_types::s3::S3Object = serde_json::from_value(raw).map_err(|e| { Error::ExecutionErr(format!("Invalid S3Object for arg `{}`: {e}", arg.name)) })?; - let json_text = - crate::sql_s3_input::fetch_s3object_as_json_text(client, &job.workspace_id, &s3_obj) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "Failed to fetch S3 object for arg `{}`: {e}", - arg.name - )) - })?; + let json_text = crate::sql_s3_input::fetch_s3object_as_json_text( + client, + conn, + job.id, + &job.workspace_id, + &s3_obj, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "Failed to fetch S3 object for arg `{}`: {e}", + arg.name + )) + })?; job_args.insert(arg.name.clone(), Value::String(json_text)); } diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 98c1b5bba7..71e78f92fe 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -768,7 +768,15 @@ pub async fn do_postgresql( // Materialize any `(s3object)` args into JSON text and rebind them as `jsonb` so // `otyp_to_pg_type` picks the right binding. Must run before the param map is // built below. - materialize_s3object_args(&mut sig.args, &mut pg_args, client, &job.workspace_id).await?; + let had_s3object_input = materialize_s3object_args( + &mut sig.args, + &mut pg_args, + client, + conn, + job.id, + &job.workspace_id, + ) + .await?; let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; @@ -867,7 +875,7 @@ pub async fn do_postgresql( }; let result = if run_inline { - result_f.await? + result_f.await } else { run_future_with_polling_update_job_poller( job.id, @@ -881,8 +889,9 @@ pub async fn do_postgresql( &mut Some(occupancy_metrics), Box::pin(futures::stream::once(async { 0 })), ) - .await? - }; + .await + } + .map_err(|e| map_s3object_jsonb_overflow(e, had_s3object_input))?; // Release the cache lock now that we have the result — allows the // post-query caching code below to re-acquire it if needed. @@ -977,13 +986,17 @@ async fn increment_connection_counter(database_string: &str) { /// For each `(s3object)` arg in `sig_args`: download the referenced file, decode it /// to JSON text, then rewrite the arg to bind as `jsonb`. Mutates `args_map` in place -/// so the existing bind path picks up the materialized payload. +/// so the existing bind path picks up the materialized payload. Returns whether any +/// `(s3object)` arg was materialized, so the jsonb-cap error can be rewritten. async fn materialize_s3object_args( sig_args: &mut [Arg], args_map: &mut HashMap, client: &AuthedClient, + conn: &Connection, + job_id: Uuid, workspace_id: &str, -) -> error::Result<()> { +) -> error::Result { + let mut materialized_any = false; for arg in sig_args.iter_mut() { if arg.otyp.as_deref() != Some("s3object") { continue; @@ -998,7 +1011,7 @@ async fn materialize_s3object_args( let s3_obj: S3Object = serde_json::from_value(raw).map_err(|e| { Error::ExecutionErr(format!("Invalid S3Object for arg `{}`: {e}", arg.name)) })?; - let json_text = fetch_s3object_as_json_text(client, workspace_id, &s3_obj) + let json_text = fetch_s3object_as_json_text(client, conn, job_id, workspace_id, &s3_obj) .await .map_err(|e| { Error::ExecutionErr(format!( @@ -1006,6 +1019,7 @@ async fn materialize_s3object_args( arg.name )) })?; + materialized_any = true; // Parse to a Value so `convert_val`'s Array/Object → JSONB branches bind it // correctly. A bare String would mismatch the JSONB param type. let parsed: Value = serde_json::from_str(&json_text).map_err(|e| { @@ -1018,7 +1032,38 @@ async fn materialize_s3object_args( arg.otyp = Some("jsonb".to_string()); arg.typ = Typ::Object(windmill_parser::ObjectType::new(None, Some(vec![]))); } - Ok(()) + Ok(materialized_any) +} + +/// A `(s3object)` input materializes the whole file into a single jsonb parameter, which +/// PostgreSQL caps at ~256MB (`total size of jsonb {array,object} elements exceeds the +/// maximum of 268435455 bytes`). A large input trips this with an opaque server error; +/// rewrite it into guidance pointing at DuckDB, which reads S3 natively and streams. +/// +/// The attribution is hedged: the same error can also come from SQL constructing an +/// oversized jsonb at execution time, and with several inputs we can't tell which one +/// overflowed, so we point at `(s3object)` inputs as the likely cause rather than naming +/// a specific file. The DuckDB remediation is the same either way. +fn map_s3object_jsonb_overflow(e: Error, had_s3object_input: bool) -> Error { + if !had_s3object_input { + return e; + } + let msg = e.to_string(); + // Match only the jsonb byte-size cap ("total size of jsonb ... elements exceeds the + // maximum of 268435455 bytes"), so the ~256 MB wording stays accurate. Excludes the + // element-count cap and unrelated caps like "array size exceeds the maximum allowed". + if !msg.contains("total size of jsonb") { + return e; + } + Error::ExecutionErr(format!( + "This query hit PostgreSQL's ~256 MB size limit for a single jsonb value. This is a \ + server-side database limit, not a worker-memory limit, so a larger worker will not raise \ + it. If a large `(s3object)` input is the cause: native SQL `(s3object)` inputs load the \ + whole file into one jsonb parameter and do not stream, so they only fit small files. For \ + large Parquet/CSV files, use a DuckDB script instead: it reads the file directly from S3 \ + and streams (e.g. `read_parquet(...)` / `read_csv_auto(...)`) rather than materializing \ + it.\n\nUnderlying error: {msg}", + )) } /// Parse a date string in formats produced by chrono's Display or JS frontends. @@ -2013,6 +2058,49 @@ impl FromSql<'_> for StringCollector { mod tests { use super::*; + #[test] + fn test_map_s3object_jsonb_overflow() { + let pg_err = Error::ExecutionErr( + "db error: ERROR: total size of jsonb array elements exceeds the maximum of 268435455 bytes".to_string(), + ); + let mapped = map_s3object_jsonb_overflow(pg_err, true).to_string(); + assert!(mapped.contains("256 MB")); + assert!(mapped.contains("larger worker will not")); + assert!(mapped.contains("read_csv_auto")); + assert!(mapped.contains("Underlying error")); + + // The element-count cap is not the byte-size cap, so the "256 MB" message would + // mislabel it — it must pass through unchanged. + let count_err = Error::ExecutionErr( + "number of jsonb array elements exceeds the maximum of 268435455".to_string(), + ); + assert_eq!( + map_s3object_jsonb_overflow(count_err, true).to_string(), + "number of jsonb array elements exceeds the maximum of 268435455", + ); + + // A different "exceeds the maximum" error must NOT be reclassified as jsonb overflow. + let array_err = + Error::ExecutionErr("array size exceeds the maximum allowed (134217727)".to_string()); + assert_eq!( + map_s3object_jsonb_overflow(array_err, true).to_string(), + "array size exceeds the maximum allowed (134217727)", + ); + + let other = Error::ExecutionErr("syntax error at or near \"SELCT\"".to_string()); + assert_eq!( + map_s3object_jsonb_overflow(other, true).to_string(), + "syntax error at or near \"SELCT\"", + ); + + // No `(s3object)` input → even a matching error is left alone. + let pg_err2 = Error::ExecutionErr("jsonb array elements exceeds the maximum".to_string()); + assert_eq!( + map_s3object_jsonb_overflow(pg_err2, false).to_string(), + "jsonb array elements exceeds the maximum", + ); + } + #[test] fn test_parse_naive_date() { // chrono's NaiveDate::to_string() format diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 9ca79c6852..13e3749fbd 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -155,16 +155,17 @@ use windmill_object_store::OBJECT_STORE_SETTINGS; use crate::{ common::{ build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, - read_result, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process, - OccupancyMetrics, StreamNotifier, DEV_CONF_NSJAIL, + read_result, render_nsjail_rlimit_as, resolve_nsjail_timeout, + resolve_nsjail_tmp_mount_block, start_child_process, OccupancyMetrics, StreamNotifier, + DEV_CONF_NSJAIL, }, get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, - UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, NSJAIL_PY_RLIMIT_AS_MB, PATH_ENV, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, + TZ_ENV, UV_CACHE_DIR, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, }; use windmill_common::client::AuthedClient; @@ -1077,6 +1078,10 @@ mount {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT + .replace( + "{RLIMIT_AS}", + &render_nsjail_rlimit_as(NSJAIL_PY_RLIMIT_AS_MB.as_deref(), 4096), + ) .replace("{JOB_DIR}", job_dir) .replace("{PY_INSTALL_DIR}", &*PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) @@ -2827,11 +2832,12 @@ pub async fn handle_python_reqs( }; // Cross-process advisory lock. Best-effort: if the filesystem doesn't - // support flock we log and proceed — verify_wheel_record + job retry - // still guard correctness, just without the dedup. - #[cfg(unix)] + // support locking we log and proceed — verify_wheel_record + job retry + // still guard correctness, just without the dedup. Cross-platform + // (flock on unix, LockFileEx on windows) so agents sharing a wheel-cache + // dir on a Windows host serialize just as they do on unix. let _venv_file_lock: Option = { - use std::os::unix::io::AsRawFd; + use fs4::fs_std::FileExt; let lock_path = format!("{venv_p}.lock"); if let Some(parent) = std::path::Path::new(&lock_path).parent() { let _ = std::fs::create_dir_all(parent); @@ -2839,17 +2845,17 @@ pub async fn handle_python_reqs( match std::fs::OpenOptions::new().create(true).write(true).open(&lock_path) { Ok(f) => { // Bounded wait: a holder that crashes releases the lock (the - // kernel drops it on fd close), but a live-but-stuck holder + // OS drops it on handle close), but a live-but-stuck holder // (e.g. uv wedged on a hung mount) would otherwise block us // forever. After the cap, proceed degraded rather than hang — // verify_wheel_record + retry still guard correctness. const MAX_WAIT: std::time::Duration = std::time::Duration::from_secs(300); let waited_since = std::time::Instant::now(); loop { - match nix::fcntl::flock(f.as_raw_fd(), nix::fcntl::FlockArg::LockExclusiveNonblock) { - Ok(()) => break Some(f), - // EWOULDBLOCK == EAGAIN on Linux: another holder has the lock. - Err(nix::errno::Errno::EWOULDBLOCK) => { + match f.try_lock_exclusive() { + Ok(true) => break Some(f), + // Another holder has the lock. + Ok(false) => { if waited_since.elapsed() >= MAX_WAIT { tracing::warn!( workspace_id = %w_id, @@ -2871,7 +2877,7 @@ pub async fn handle_python_reqs( Err(e) => { tracing::warn!( workspace_id = %w_id, - "could not flock {lock_path}, proceeding without cross-process install lock: {e}" + "could not lock {lock_path}, proceeding without cross-process install lock: {e}" ); break Some(f); } @@ -3777,14 +3783,14 @@ mod tests { ); } - #[cfg(unix)] #[tokio::test] async fn test_venv_file_lock_excludes_across_descriptions() { - // The cross-process layer: flock on a sibling `.lock` excludes a second - // independent open file description (i.e. another worker process) while - // held, and frees it on close. Mirrors the loop in handle_python_reqs. - use nix::fcntl::{flock, FlockArg}; - use std::os::unix::io::AsRawFd; + // The cross-process layer: an advisory lock on a sibling `.lock` excludes a + // second independent open file handle (i.e. another worker process) while + // held, and frees it on close. Mirrors the loop in handle_python_reqs and + // must hold on every platform (flock on unix, LockFileEx on windows) — a + // Windows host running several agents against one wheel cache relies on it. + use fs4::fs_std::FileExt; let dir = std::env::temp_dir().join("wm_venv_lock_test"); std::fs::create_dir_all(&dir).unwrap(); @@ -3795,24 +3801,28 @@ mod tests { .write(true) .open(&lock_path) .unwrap(); - flock(f1.as_raw_fd(), FlockArg::LockExclusiveNonblock).unwrap(); + assert!( + f1.try_lock_exclusive().unwrap(), + "first holder must acquire the lock" + ); - // A second descriptor (stand-in for another process) cannot take it. + // A second handle (stand-in for another process) cannot take it. let f2 = std::fs::OpenOptions::new() .create(true) .write(true) .open(&lock_path) .unwrap(); - assert_eq!( - flock(f2.as_raw_fd(), FlockArg::LockExclusiveNonblock), - Err(nix::errno::Errno::EWOULDBLOCK), + assert!( + !f2.try_lock_exclusive().unwrap(), "a second holder must be blocked while the lock is held" ); // Releasing the first lets the second acquire it. drop(f1); - flock(f2.as_raw_fd(), FlockArg::LockExclusiveNonblock) - .expect("lock must be acquirable once the holder releases it"); + assert!( + f2.try_lock_exclusive().unwrap(), + "lock must be acquirable once the holder releases it" + ); drop(f2); let _ = std::fs::remove_file(&lock_path); diff --git a/backend/windmill-worker/src/r_executor.rs b/backend/windmill-worker/src/r_executor.rs index 0d4219c0df..9b29ae35b6 100644 --- a/backend/windmill-worker/src/r_executor.rs +++ b/backend/windmill-worker/src/r_executor.rs @@ -30,7 +30,7 @@ use crate::{ par_install_language_dependencies_seq, DependencyGraph, InstallDeps, RequiredDependency, }, DISABLE_NUSER, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, R_CACHE_DIR, - TRACING_PROXY_CA_CERT_PATH, + TRACING_PROXY_CA_CERT_PATH, WIN_ENVS, }; use windmill_common::scripts::ScriptLang; @@ -518,6 +518,9 @@ async fn install<'a>( cmd.env_clear() .current_dir(&job_dir) .env("PATH", PATH_ENV.as_str()) + // On Windows, renv needs SystemRoot (winsock init — without it DNS + // and sockets fail) and LOCALAPPDATA (its cache root) to install. + .envs(WIN_ENVS.to_vec()) .envs(R_PROXY_ENVS.clone()); cmd .args(&[ @@ -694,15 +697,7 @@ async fn run<'a>( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(windows)] - { - cmd.env("SystemRoot", crate::SYSTEM_ROOT.as_str()) - .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ); - } + cmd.envs(WIN_ENVS.to_vec()); start_child_process(cmd, rscript_executable, false).await? }; handle_child::handle_child( diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index f2f36621a5..c95bd70899 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -741,6 +741,679 @@ pub async fn handle_receive_completed_job( } } +/// A git-sync check run threaded through a pull job: the PR diff preview (phase 4) +/// or the live deploy status (phase 6). Both markers carry the same shape. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[derive(serde::Deserialize)] +struct GitSyncCheck { + check_run_id: i64, + repo_url: String, + #[serde(default)] + pr_number: Option, + #[serde(default)] + head_sha: Option, + /// Whether the PR itself modifies wmill.yaml (None = undetermined); picks + /// the wording for a settings difference in the diff summary. + #[serde(default)] + wmill_yaml_changed: Option, +} + +/// Parsed diff summary from a (dry-run or real) pull result. `None` when the +/// result can't be parsed into the expected shape. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn parse_git_sync_changes(result_raw: &str) -> Option<(Vec<(String, String)>, bool)> { + use serde::Deserialize; + #[derive(Deserialize)] + struct Change { + #[serde(rename = "type")] + change_type: String, + path: String, + } + #[derive(Deserialize)] + struct SettingsDiff { + #[serde(rename = "hasChanges", default)] + has_changes: bool, + } + #[derive(Deserialize)] + struct SyncResponse { + changes: Option>, + #[serde(rename = "settingsDiffResult")] + settings_diff_result: Option, + } + let resp = serde_json::from_str::(result_raw).ok()?; + // A result carrying neither field isn't a recognizable diff; return None so the + // caller falls back to the unsummarized path instead of a false "in sync". + if resp.changes.is_none() && resp.settings_diff_result.is_none() { + return None; + } + let settings_changed = resp + .settings_diff_result + .map(|s| s.has_changes) + .unwrap_or(false); + Some(( + resp.changes + .unwrap_or_default() + .into_iter() + .map(|c| (c.change_type, c.path)) + .collect(), + settings_changed, + )) +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +fn format_change_list(changes: &[(String, String)]) -> Vec { + let mut lines = Vec::new(); + for (change_type, path) in changes.iter().take(100) { + lines.push(format!("- `{}` {}", change_type, path)); + } + if changes.len() > 100 { + lines.push(format!("- ... and {} more", changes.len() - 100)); + } + lines +} + +#[cfg(all(test, feature = "enterprise", feature = "private"))] +mod git_sync_check_tests { + use super::{format_change_list, parse_git_sync_changes}; + + #[test] + fn parse_empty_changes_is_in_sync() { + // Present-but-empty diff → a real "in sync" result, not None. + let (changes, settings) = parse_git_sync_changes(r#"{"changes":[]}"#).unwrap(); + assert!(changes.is_empty()); + assert!(!settings); + } + + #[test] + fn parse_missing_fields_is_none() { + // Neither field present → unrecognizable, falls back to the caller's path. + assert!(parse_git_sync_changes("{}").is_none()); + } + + #[test] + fn parse_unparseable_is_none() { + assert!(parse_git_sync_changes("not json").is_none()); + } + + #[test] + fn parse_changes_and_settings() { + let (changes, settings) = parse_git_sync_changes( + r#"{"changes":[{"type":"edited","path":"f/a"}],"settingsDiffResult":{"hasChanges":true}}"#, + ) + .unwrap(); + assert_eq!(changes, vec![("edited".to_string(), "f/a".to_string())]); + assert!(settings); + } + + #[test] + fn format_truncates_over_100() { + let changes: Vec<(String, String)> = (0..150) + .map(|i| ("edited".to_string(), format!("f/{i}"))) + .collect(); + let lines = format_change_list(&changes); + assert_eq!(lines.len(), 101); + assert_eq!(lines.last().unwrap(), "- ... and 50 more"); + } +} + +/// When an auto-pull job (carrying `__git_sync_auto_pull`) fails, roll the +/// optimistic `last_synced_sha` advance back to the pre-pull value so the commit +/// is retried instead of being silently treated as synced, and record the failure. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_reconcile_git_sync_auto_pull( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + success: bool, +) { + if success { + return; // the optimistic synced state is already correct + } + let marker: Option = match sqlx::query_scalar!( + "SELECT args->'__git_sync_auto_pull' FROM v2_job WHERE id = $1", + job_id + ) + .fetch_optional(db) + .await + { + Ok(v) => v.flatten(), + Err(e) => { + tracing::error!("git auto-pull: failed to read job args: {e:#}"); + return; + } + }; + let Some(marker) = marker else { + return; + }; + #[derive(serde::Deserialize)] + struct AutoPullMarker { + repo_resource_path: String, + #[serde(default)] + prev_synced: std::collections::HashMap, + } + let Ok(m) = serde_json::from_value::(marker) else { + return; + }; + windmill_git_sync::record_auto_pull_failure( + db, + workspace_id, + &m.repo_resource_path, + &m.prev_synced, + "auto-pull job failed".to_string(), + ) + .await; +} + +/// Branch a git-sync push job deployed to, mirroring the hub script's +/// derivation: a dev workspace deploys to its environment-label branch +/// (`dev`/`staging`), other fork workspaces to `wm-fork//`, +/// else the promotion `wm_deploy/**` formula (per-folder or per-item form). +/// A dev workspace in promotion mode is the exception: it takes the promotion +/// `wm_deploy/**` formula (per-item PRs into the parent) instead of its label +/// branch. `None` when the deploy stays on the base branch (workspace-wide +/// mode) and has no PR to open. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn git_sync_deploy_pr_head_branch( + workspace_id: &str, + parent_workspace_id: Option<&str>, + dev_workspace_label: Option<&str>, + base: &str, + use_individual_branch: bool, + group_by_folder: bool, + item_path: &str, + item_parent_path: &str, + path_type: &str, +) -> Option { + let is_dev = dev_workspace_label.is_some(); + // A dev workspace with promotion on falls through to the wm_deploy/** + // formula below; the label/fork branches only apply when promotion is off. + if !(is_dev && use_individual_branch) { + if is_dev { + return Some(windmill_common::workspaces::dev_workspace_branch( + dev_workspace_label, + )); + } + let is_fork = parent_workspace_id.is_some() + || workspace_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX); + if is_fork { + let suffix = workspace_id + .strip_prefix(windmill_common::workspaces::WM_FORK_PREFIX) + .unwrap_or(workspace_id); + return Some(format!("wm-fork/{base}/{suffix}")); + } + } + if !use_individual_branch { + return None; + } + // Mirrors the CLI's computeGitSyncDeployBranch: user/group objects are + // pushed to the base branch and never get their own wm_deploy branch. + if path_type == "user" || path_type == "group" { + return None; + } + let git_ref = if !item_path.is_empty() { + item_path + } else { + item_parent_path + }; + if git_ref.is_empty() { + return None; + } + Some(if group_by_folder { + format!( + "wm_deploy/{workspace_id}/{}", + git_ref.split('/').take(2).collect::>().join("__") + ) + } else { + format!( + "wm_deploy/{workspace_id}/{}/{}", + path_type, + git_ref.replace('/', "__") + ) + }) +} + +/// Whether the push job's result says a commit was actually pushed. `None` +/// when the result doesn't carry the flag (hub script versions predating it). +#[cfg(all(feature = "enterprise", feature = "private"))] +fn git_sync_push_result_pushed(result: &str) -> Option { + serde_json::from_str::(result) + .ok()? + .get("pushed")? + .as_bool() +} + +/// When a git-sync push job carrying `__git_sync_open_pr` succeeds, open (or +/// reopen) the PR for the branch it pushed: `wm-fork//` for a fork +/// deploy, `wm_deploy/**` for a promotion deploy. Runs outbound with the +/// installation token, so it works regardless of webhook reachability. +/// Best-effort: failures are logged, never propagated. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_open_git_sync_deploy_pr( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + result: &str, +) { + // A no-op push (workspace already matches the repo — e.g. the deploy was + // itself caused by an auto-pull) must not ensure a PR: it would recreate + // PRs the user closed and spam creation attempts with no diff. + if git_sync_push_result_pushed(result) == Some(false) { + return; + } + let row = match sqlx::query!( + r#"SELECT + args->'__git_sync_open_pr' as "marker", + args->>'repo_url_resource_path' as "repo_path", + args->>'parent_workspace_id' as "parent_workspace_id", + args->>'dev_workspace_label' as "dev_workspace_label", + args->>'parent_dev_workspace_label' as "parent_dev_workspace_label", + COALESCE((args->'use_individual_branch')::bool, false) as "use_individual_branch!", + COALESCE((args->'group_by_folder')::bool, false) as "group_by_folder!", + COALESCE(args->'items'->0->>'path', args->>'path', '') as "item_path!", + COALESCE(args->'items'->0->>'parent_path', args->>'parent_path', '') as "item_parent_path!", + COALESCE(args->'items'->0->>'path_type', args->>'path_type', '') as "path_type!", + COALESCE(args->'items'->0->>'commit_msg', args->>'commit_msg', '') as "commit_msg!" + FROM v2_job WHERE id = $1"#, + job_id + ) + .fetch_optional(db) + .await + { + Ok(Some(r)) => r, + Ok(None) => return, + Err(e) => { + tracing::error!("git sync PR: failed to read job args: {e:#}"); + return; + } + }; + if row.marker.is_none() { + return; + } + // Runtime Enterprise gate, like the poller: the toggles may have been set + // while a license was active (or written directly), and this hook drives + // GitHub API calls with the installation token. + if !matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) { + tracing::warn!( + "git sync PR: skipping PR creation for {workspace_id}: requires an Enterprise license" + ); + return; + } + let Some(repo_path) = row.repo_path else { + return; + }; + + // Base = the tracked branch (resource branch, else the repo default). Also + // acts as the app-backed gate: PR creation needs the installation token. + let base = match windmill_common::git_sync_ee::get_app_repo_head_for_autopull( + db, + workspace_id, + &repo_path, + ) + .await + { + Ok(Some((branch, _))) => branch, + Ok(None) => { + tracing::warn!( + "git sync PR: repo {repo_path} in {workspace_id} has a PR-on-deploy toggle set but is not GitHub-App-backed; skipping (connect the repo through the GitHub App, or use the open-pr-on-commit workflow)" + ); + return; + } + Err(e) => { + tracing::warn!("git sync PR: could not resolve base branch for {repo_path}: {e:#}"); + return; + } + }; + + let Some(head) = git_sync_deploy_pr_head_branch( + workspace_id, + row.parent_workspace_id.as_deref(), + row.dev_workspace_label.as_deref(), + &base, + row.use_individual_branch, + row.group_by_folder, + &row.item_path, + &row.item_parent_path, + &row.path_type, + ) else { + return; + }; + + let repo_url = match windmill_common::git_sync_ee::resolve_repo_url_interpolated( + db, + workspace_id, + &repo_path, + ) + .await + { + Ok(url) => url, + Err(e) => { + tracing::warn!("git sync PR: could not resolve repo url for {repo_path}: {e:#}"); + return; + } + }; + // A fork of a dev workspace diverged from the dev's label branch, so its PR + // merges back there; everything else targets the tracked branch. + let pr_base = row.parent_dev_workspace_label.as_deref().unwrap_or(&base); + match windmill_common::git_sync_ee::ensure_pull_request( + db, + workspace_id, + &repo_url, + &head, + pr_base, + &row.commit_msg, + ) + .await + { + Ok(()) => { + persist_git_sync_open_pr_error(db, workspace_id, &repo_path, None).await; + } + Err(e) => { + tracing::warn!( + "git sync PR: failed to open PR {head} -> {pr_base} for {repo_path}: {e:#}" + ); + let msg: String = format!("{e:#}").chars().take(400).collect(); + persist_git_sync_open_pr_error(db, workspace_id, &repo_path, Some(msg)).await; + } + } +} + +/// Best-effort: record (or clear) the last PR-creation failure on the repo's +/// settings so the toggle can explain a silent no-op in the UI (the usual +/// cause is a GitHub App installation that hasn't approved the pull-request +/// permission yet). Merges into a fresh read of the row, only writes on change. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn persist_git_sync_open_pr_error( + db: &DB, + workspace_id: &str, + repo_path: &str, + error: Option, +) { + // Single targeted update (like the EE auto-pull status writer): a full + // read-modify-write of the column would race the poller's concurrent + // last_synced_sha/last_pull_status writes and silently clobber them. + let bare_path = repo_path.trim_start_matches("$res:"); + let result: error::Result<()> = async { + sqlx::query!( + r#" + UPDATE workspace_settings + SET git_sync = jsonb_set( + git_sync, + '{repositories}', + (SELECT jsonb_agg( + CASE WHEN elem->>'git_repo_resource_path' IN ($2, '$res:' || $2) + THEN CASE WHEN $3::text IS NULL THEN elem - 'open_pr_error' + ELSE jsonb_set(elem, '{open_pr_error}', to_jsonb($3::text), true) END + ELSE elem END) + FROM jsonb_array_elements(git_sync->'repositories') AS elem) + ) + WHERE workspace_id = $1 + AND jsonb_typeof(git_sync->'repositories') = 'array' + "#, + workspace_id, + bare_path, + error.as_deref(), + ) + .execute(db) + .await?; + Ok(()) + } + .await; + if let Err(e) = result { + tracing::warn!("git sync PR: failed to persist open_pr_error for {repo_path}: {e:#}"); + } +} + +/// When a git-sync pull job carrying a check marker completes, post the outcome +/// to its GitHub check run: the PR diff preview (`__git_sync_pr_check`, phase 4) +/// or the live deploy status (`__git_sync_deploy_check`, phase 6). +/// A one-line description of the repo's sync filters, appended to an "in sync" +/// PR verdict: a PR that only touches files outside these paths deploys +/// nothing on merge, which otherwise looks like a wrong verdict. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn format_git_sync_scope_note(include: &[String], exclude: &[String]) -> Option { + if include.is_empty() { + return None; + } + let fmt = |paths: &[String]| { + paths + .iter() + .map(|p| format!("`{p}`")) + .collect::>() + .join(", ") + }; + let mut note = format!( + "\n\nOnly files matching this repository's sync filters deploy on merge: {}", + fmt(include) + ); + if !exclude.is_empty() { + note.push_str(&format!(" (excluding {})", fmt(exclude))); + } + note.push('.'); + Some(note) +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn git_sync_repo_scope_note( + db: &DB, + workspace_id: &str, + repo_path: Option<&str>, +) -> Option { + let repo_path = repo_path?; + let settings = sqlx::query_scalar!( + "SELECT git_sync FROM workspace_settings WHERE workspace_id = $1", + workspace_id + ) + .fetch_optional(db) + .await + .ok()? + .flatten()?; + let settings: windmill_common::workspaces::WorkspaceGitSyncSettings = + serde_json::from_value(settings).ok()?; + // Job args carry the bare resource path; stored settings keep the $res: prefix. + let repo = settings.repositories.iter().find(|r| { + r.git_repo_resource_path.trim_start_matches("$res:") + == repo_path.trim_start_matches("$res:") + })?; + let s = repo.settings.as_ref()?; + format_git_sync_scope_note( + &s.include_path, + s.exclude_path.as_deref().unwrap_or_default(), + ) +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn maybe_post_git_sync_check( + db: &DB, + job_id: &uuid::Uuid, + workspace_id: &str, + success: bool, + result_raw: &str, +) { + // Only git-sync pull jobs carry one of these markers; everything else no-ops. + let row = match sqlx::query!( + r#"SELECT args->'__git_sync_pr_check' AS "pr", args->'__git_sync_deploy_check' AS "deploy", + args->>'repo_url_resource_path' AS "repo_path" + FROM v2_job WHERE id = $1"#, + job_id + ) + .fetch_optional(db) + .await + { + Ok(r) => r, + Err(e) => { + tracing::error!("git sync-check: failed to read job args: {e:#}"); + return; + } + }; + let Some(row) = row else { + return; + }; + // A PR dry-run and a deploy pull are mutually exclusive markers. + let (is_deploy, marker) = match (row.pr, row.deploy) { + (Some(pr), _) => (false, pr), + (None, Some(deploy)) => (true, deploy), + (None, None) => return, + }; + let Ok(mut check) = serde_json::from_value::(marker) else { + return; + }; + // Markers carry the literal resource URL (job args are persisted, so a + // `$var:`-resolved URL must not land there); interpolate before calling + // GitHub. + check.repo_url = + match windmill_common::variables::get_variable_or_self(check.repo_url, db, workspace_id) + .await + { + Ok(u) => u, + Err(e) => { + tracing::error!("git sync-check: cannot interpolate repo url: {e:#}"); + return; + } + }; + // "In sync" on a PR that visibly changes files reads as a bug when those + // files are outside the repo's sync filters — say what the scope is. + let scope_note = if !is_deploy && success { + git_sync_repo_scope_note(db, workspace_id, row.repo_path.as_deref()).await + } else { + None + }; + + let (conclusion, title, summary): (&str, String, String) = if is_deploy { + // Phase 6: real deploy pull -> "Deployed N changes" / "In sync" / failure. + if !success { + ( + "failure", + format!("Deploy to {} failed", workspace_id), + "Deploying the latest commit failed. See the job in Windmill for details." + .to_string(), + ) + } else { + match parse_git_sync_changes(result_raw) { + Some((changes, settings_changed)) if changes.is_empty() && !settings_changed => ( + "success", + format!("In sync with {}", workspace_id), + format!( + "No changes to deploy to `{}` from this commit.", + workspace_id + ), + ), + Some((changes, settings_changed)) => { + let mut lines = vec![format!( + "Deployed {} change(s) to `{}`:\n", + changes.len(), + workspace_id + )]; + lines.extend(format_change_list(&changes)); + if settings_changed { + lines.push("\nWorkspace settings also changed.".to_string()); + } + ( + "success", + format!("Deployed {} change(s) to {}", changes.len(), workspace_id), + lines.join("\n"), + ) + } + None => ( + "success", + format!("Deployed to {}", workspace_id), + format!("Windmill deployed the latest commit to `{}`.", workspace_id), + ), + } + } + } else { + // Phase 4: dry-run diff preview for a PR. + if !success { + ( + "failure", + "Windmill diff failed".to_string(), + "The dry-run pull to compute the diff failed. See the job in Windmill for details." + .to_string(), + ) + } else { + match parse_git_sync_changes(result_raw) { + Some((changes, settings_changed)) if changes.is_empty() && !settings_changed => ( + "success", + "In sync".to_string(), + format!( + "Merging this PR would make no changes to the workspace.{}", + scope_note.as_deref().unwrap_or_default() + ), + ), + Some((changes, settings_changed)) => { + let mut lines = vec![format!( + "Merging this PR would apply {} change(s) to the workspace:\n", + changes.len() + )]; + lines.extend(format_change_list(&changes)); + if settings_changed { + lines.push(match check.wmill_yaml_changed { + Some(true) => "\nThis PR changes wmill.yaml: pulling also applies the updated workspace settings.".to_string(), + Some(false) => "\nIndependent of this PR, the workspace's git-sync settings differ from the repo's wmill.yaml and a pull updates them to match.".to_string(), + None => "\nA pull also updates the workspace's git-sync settings to match the repo's wmill.yaml.".to_string(), + }); + } + ( + "neutral", + format!("{} change(s) to deploy", changes.len()), + lines.join("\n"), + ) + } + None => ( + "neutral", + "Diff computed".to_string(), + "Windmill computed a diff but could not summarize it.".to_string(), + ), + } + } + }; + + if let Err(e) = windmill_common::git_sync_ee::update_check_run( + db, + workspace_id, + &check.repo_url, + check.check_run_id, + conclusion, + &title, + &summary, + ) + .await + { + tracing::error!("git sync-check: failed to update check run: {e:#}"); + } + + // Phase 4 also maintains ONE managed comment on the PR (Cloudflare + // deploy-preview style): upserted on every synchronize, so reviewers see the + // current diff without opening the Checks tab. + if !is_deploy { + if let Some(pr_number) = check.pr_number { + let marker = ""; + let head = check + .head_sha + .as_deref() + .map(|s| &s[..s.len().min(7)]) + .unwrap_or("latest"); + let body = format!( + "{marker}\n### Windmill deploy preview\n\n| | |\n|---|---|\n| **Workspace** | `{workspace_id}` |\n| **Status** | {title} |\n| **Commit** | `{head}` |\n\n
Details\n\n{summary}\n\n
" + ); + if let Err(e) = windmill_common::git_sync_ee::upsert_pr_comment( + db, + workspace_id, + &check.repo_url, + pr_number, + marker, + &body, + ) + .await + { + tracing::warn!("git sync-check: failed to upsert PR diff comment: {e:#}"); + } + } + } +} + pub async fn process_completed_job( JobCompleted { job, @@ -826,6 +1499,11 @@ pub async fn process_completed_job( from_cache.unwrap_or(false), ) .await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + if job.kind == JobKind::DeploymentCallback { + maybe_post_git_sync_check(db, &job_id, &workspace_id, true, result.get()).await; + maybe_open_git_sync_deploy_pr(db, &job_id, &workspace_id, result.get()).await; + } // Asset-trigger fan-out: best-effort, never propagates errors. // Internal eligibility checks gate to top-level Script/Preview runs; @@ -933,6 +1611,11 @@ pub async fn process_completed_job( .await?; Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()) }; + #[cfg(all(feature = "enterprise", feature = "private"))] + if job.kind == JobKind::DeploymentCallback { + maybe_post_git_sync_check(db, &job.id, &job.workspace_id, false, result.get()).await; + maybe_reconcile_git_sync_auto_pull(db, &job.id, &job.workspace_id, false).await; + } if job.is_flow_step() { if let Some(parent_job) = job.parent_job { tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status"); @@ -1359,3 +2042,280 @@ pub fn extract_error_value( exit_code: Some(i), }); } + +#[cfg(all(test, feature = "enterprise", feature = "private"))] +mod git_sync_pr_tests { + use super::{git_sync_deploy_pr_head_branch, git_sync_push_result_pushed}; + + #[test] + fn user_and_group_items_get_no_promotion_branch() { + for path_type in ["user", "group"] { + assert_eq!( + git_sync_deploy_pr_head_branch( + "ws", + None, + None, + "main", + true, + false, + "u/someone", + "", + path_type + ), + None + ); + } + } + + #[test] + fn scope_note_lists_filters() { + use super::format_git_sync_scope_note; + assert_eq!(format_git_sync_scope_note(&[], &[]), None); + assert_eq!( + format_git_sync_scope_note(&["f/**".into()], &[]).unwrap(), + "\n\nOnly files matching this repository's sync filters deploy on merge: `f/**`." + ); + assert_eq!( + format_git_sync_scope_note(&["f/**".into(), "u/**".into()], &["f/pat/**".into()]) + .unwrap(), + "\n\nOnly files matching this repository's sync filters deploy on merge: `f/**`, `u/**` (excluding `f/pat/**`)." + ); + } + + #[test] + fn push_result_pushed_flag() { + assert_eq!( + git_sync_push_result_pushed(r#"{"pushed": true}"#), + Some(true) + ); + assert_eq!( + git_sync_push_result_pushed(r#"{"pushed": false}"#), + Some(false) + ); + // Older hub script versions return null / no flag: undetermined. + assert_eq!(git_sync_push_result_pushed("null"), None); + assert_eq!(git_sync_push_result_pushed(r#"{"other": 1}"#), None); + assert_eq!(git_sync_push_result_pushed("not json"), None); + } + + #[test] + fn fork_branch_wins_and_strips_the_id_prefix() { + // Generated fork id: branch suffix drops the wm-fork- prefix. + assert_eq!( + git_sync_deploy_pr_head_branch( + "wm-fork-abc", + Some("prod"), + None, + "main", + false, + false, + "", + "", + "" + ), + Some("wm-fork/main/abc".to_string()) + ); + // Dev workspace (prefix-less id, detected via parent): verbatim suffix. + assert_eq!( + git_sync_deploy_pr_head_branch( + "staging", + Some("prod"), + None, + "main", + false, + false, + "", + "", + "" + ), + Some("wm-fork/main/staging".to_string()) + ); + // Orphaned fork (parent deleted): the id prefix still identifies it. + assert_eq!( + git_sync_deploy_pr_head_branch( + "wm-fork-abc", + None, + None, + "main", + true, + false, + "f/x/y", + "", + "script" + ), + Some("wm-fork/main/abc".to_string()) + ); + } + + #[test] + fn promotion_branch_matches_the_hub_script_formula() { + // Per-item form: wm_deploy/// __>. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + None, + None, + "main", + true, + false, + "f/folder/my_script", + "", + "script" + ), + Some("wm_deploy/dev/script/f__folder__my_script".to_string()) + ); + // Grouped-by-folder form: first two path segments joined by __. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + None, + None, + "main", + true, + true, + "f/folder/my_script", + "", + "script" + ), + Some("wm_deploy/dev/f__folder".to_string()) + ); + // Renamed object: falls back to the parent path. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + None, + None, + "main", + true, + false, + "", + "f/folder/old", + "script" + ), + Some("wm_deploy/dev/script/f__folder__old".to_string()) + ); + } + + #[test] + fn no_branch_when_deploy_stays_on_base() { + // Workspace-wide mode commits straight to the tracked branch. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", None, None, "main", false, false, "f/x/y", "", "script" + ), + None + ); + // Promotion mode but no per-item ref (e.g. user/group objects). + assert_eq!( + git_sync_deploy_pr_head_branch("dev", None, None, "main", true, false, "", "", "user"), + None + ); + } + + #[test] + fn dev_workspace_label_branch_wins() { + // Dev workspaces deploy to their environment-label branch verbatim. + assert_eq!( + git_sync_deploy_pr_head_branch( + "staging-ws", + Some("prod"), + Some("staging"), + "main", + false, + false, + "", + "", + "" + ), + Some("staging".to_string()) + ); + // Label present even on a wm-fork-prefixed id: label still wins. + assert_eq!( + git_sync_deploy_pr_head_branch( + "wm-fork-x", + Some("prod"), + Some("dev"), + "main", + false, + false, + "", + "", + "" + ), + Some("dev".to_string()) + ); + } + + #[test] + fn dev_workspace_promotion_uses_wm_deploy_branch() { + // Promotion on: a dev workspace gets per-item wm_deploy/** branches + // (namespaced by its own id), not its env-label branch. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + Some("prod"), + Some("dev"), + "main", + true, + false, + "f/folder/my_script", + "", + "script" + ), + Some("wm_deploy/dev/script/f__folder__my_script".to_string()) + ); + // Per-folder form still honored for a promotion dev workspace. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + Some("prod"), + Some("dev"), + "main", + true, + true, + "f/folder/my_script", + "", + "script" + ), + Some("wm_deploy/dev/f__folder".to_string()) + ); + // Promotion off: the env-label branch still wins. + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + Some("prod"), + Some("dev"), + "main", + false, + false, + "f/x/y", + "", + "script" + ), + Some("dev".to_string()) + ); + } + + #[test] + fn dev_promotion_user_group_items_open_no_pr() { + // User/group objects get no wm_deploy branch even on a dev workspace; the + // CLI isolates them to the env-label branch, so the backend opens no PR + // (never a PR from the env-label branch into the parent for these). + for path_type in ["user", "group"] { + assert_eq!( + git_sync_deploy_pr_head_branch( + "dev", + Some("prod"), + Some("dev"), + "main", + true, + false, + "u/alice", + "", + path_type + ), + None + ); + } + } +} diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index d3d4555f40..b4bb448b99 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -567,6 +567,8 @@ pub async fn do_snowflake( })?; let json_text = crate::sql_s3_input::fetch_s3object_as_json_text( client, + conn, + job.id, &job.workspace_id, &s3_obj, ) @@ -718,8 +720,6 @@ pub async fn do_snowflake( ) }; - tracing::debug!("Snowflake token: {}", token); - let mut body = serde_json::Map::new(); if database.schema.is_some() { body.insert( diff --git a/backend/windmill-worker/src/sql_s3_input.rs b/backend/windmill-worker/src/sql_s3_input.rs index 937a21fa20..e3a5b25411 100644 --- a/backend/windmill-worker/src/sql_s3_input.rs +++ b/backend/windmill-worker/src/sql_s3_input.rs @@ -12,6 +12,7 @@ use anyhow::Context; use windmill_common::client::AuthedClient; +use windmill_common::worker::Connection; use windmill_types::s3::S3Object; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -44,9 +45,16 @@ fn detect_format(key: &str) -> InputFormat { /// uniformly assume an array shape. pub async fn fetch_s3object_as_json_text( client: &AuthedClient, + conn: &Connection, + job_id: uuid::Uuid, workspace_id: &str, obj: &S3Object, ) -> anyhow::Result { + // This runs before the executor's polling loop starts, so heartbeat the job ping + // to keep a slow download/decode from tripping the zombie monitor. + let _ping_heartbeat = + crate::worker_utils::JobPingHeartbeat::start(conn, job_id, "s3object materialization"); + let bytes = client .download_s3_file(workspace_id, &obj.s3, obj.storage.clone()) .await diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 2fcfc8e2c5..e3eedac36b 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -143,14 +143,53 @@ pub async fn prepare_checkpoint_for_resume( .and_then(|p| p.keys.first().cloned()) .unwrap_or_default(); - let resume_row = sqlx::query_as::<_, (sqlx::types::Json>, Option, bool)>( - "SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1", + // Exclude rows already consumed by earlier approvals so each step + // reads its own (rows are never deleted). resume_id can't key this: + // it's only hash(step_key) for the inline URL, while the approval + // page, in-run button, Slack, Teams and resume-as-owner store a + // random id. A timed-out step matches no row -> else branch below. + let consumed = checkpoint.consumed_resume_row_ids.clone(); + + // A row carrying another step's bound resume_id answers that step, not + // this one, so it must never be picked up here however it got in — the + // API rejects such resumes but cannot do so atomically with the insert. + // Only keys this workflow minted a URL for are known to be bound; every + // other resume_id stays eligible, preserving WIN-2241 for the channels + // that sign random ids. + let foreign_bound_ids: Vec = sqlx::query_scalar::<_, String>( + "SELECT jsonb_object_keys( + COALESCE(workflow_as_code_status->'_minted_approval_keys', '{}'::jsonb)) + FROM v2_job_status WHERE id = $1", ) .bind(job_id) + .fetch_all(db) + .await? + .into_iter() + .filter(|k| *k != approval_key) + .map(|k| windmill_common::wac::approval_resume_id(&k) as i32) + .collect(); + + let resume_row = sqlx::query_as::< + _, + ( + Uuid, + sqlx::types::Json>, + Option, + bool, + ), + >( + "SELECT id, value, approver, approved FROM resume_job \ + WHERE job = $1 AND id <> ALL($2) AND resume_id <> ALL($3) \ + ORDER BY created_at ASC LIMIT 1", + ) + .bind(job_id) + .bind(&consumed) + .bind(&foreign_bound_ids) .fetch_optional(db) .await?; - let approval_result = if let Some((value, approver, approved)) = resume_row { + let approval_result = if let Some((row_id, value, approver, approved)) = resume_row { + checkpoint.consumed_resume_row_ids.push(row_id); serde_json::json!({ "value": serde_json::from_str::(value.get()).unwrap_or(Value::Null), "approver": approver.unwrap_or_else(|| "anonymous".to_string()), diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a4672145bb..ab7051af08 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -149,7 +149,7 @@ use crate::{ pwsh_executor::handle_powershell_job, result_processor::{handle_job_error, process_result, start_background_processor}, schema::schema_validator_from_main_arg_sig, - worker_flow::{handle_flow, SchedulePushZombieError}, + worker_flow::handle_flow, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, }, @@ -194,13 +194,13 @@ use crate::oracledb_executor::do_oracledb; #[cfg(all(feature = "private", feature = "enterprise"))] use crate::dedicated_worker_oss::create_dedicated_worker_map; -#[cfg(feature = "enterprise")] +#[cfg(feature = "snowflake")] use crate::snowflake_executor::do_snowflake; #[cfg(all(feature = "enterprise", feature = "mssql"))] use crate::mssql_executor::do_mssql; -#[cfg(all(feature = "enterprise", feature = "bigquery"))] +#[cfg(feature = "bigquery")] use crate::bigquery_executor::do_bigquery; #[cfg(feature = "benchmark")] @@ -283,6 +283,17 @@ pub struct OtelTracingProxySettings { pub enabled_languages: HashSet, #[serde(default)] pub no_proxy_hosts: Option, + /// Comma-separated host/IP patterns for which the MITM proxy skips upstream TLS + /// verification. Unlike `no_proxy_hosts` (which bypasses the proxy entirely, so the + /// request goes untraced), these hosts stay traced — only the proxy's own upstream + /// certificate check is disabled. Same suffix-matching semantics as `no_proxy_hosts`. + #[serde(default)] + pub insecure_upstream_hosts: Option, + /// Extra CA certificates (PEM bundle) added to the MITM proxy's upstream trust store, + /// on top of the system roots. Lets the proxy verify internal endpoints signed by a + /// private CA without disabling verification. + #[serde(default)] + pub upstream_ca_certs: Option, } #[cfg(feature = "prometheus")] @@ -342,6 +353,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + /// Per-language override for the nsjail `rlimit_as` (virtual address space) cap. + /// Value is in MiB, or `unlimited`/`none`/`inf`/`0` to uncap. Unset keeps the + /// historical default baked into the proto. See `render_nsjail_rlimit_as`. + pub static ref NSJAIL_PY_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_PY_RLIMIT_AS_MB").ok(); + pub static ref NSJAIL_ANSIBLE_RLIMIT_AS_MB: Option = + std::env::var("NSJAIL_ANSIBLE_RLIMIT_AS_MB").ok(); + // pub static ref DISABLE_NSJAIL: bool = false; pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL") .ok() @@ -1583,7 +1602,7 @@ const STATUS_DESCRIPTION_MAX_LEN: usize = 512; #[derive(Debug)] pub enum JobOutcome { /// Job ran cleanly, was forwarded as a flow, was a no-op (test workspace), - /// or was suspended waiting for child jobs (WAC v2 / schedule zombie). + /// or was suspended waiting for child jobs (WAC v2). /// All of these leave the span `Status` `Unset`. Completed, /// Job was attempted but its execution returned an error; the failure has @@ -1998,6 +2017,9 @@ pub async fn run_worker( create_directory_async(&worker_dir).await; + #[cfg(all(feature = "python", unix))] + crate::ansible_executor::prepare_persistent_control_path_root().await; + if is_sandboxing_enabled() { let _ = write_file( &worker_dir, @@ -3797,7 +3819,7 @@ pub async fn handle_queued_job( // Not a preview: fetch from the cache or the database. _ => cache::job::fetch_flow(db, &job.kind, job.runnable_id).await?, }; - match Box::pin(handle_flow( + Box::pin(handle_flow( job, &flow_data, db, @@ -3814,19 +3836,8 @@ pub async fn handle_queued_job( false, )) .warn_after_seconds(10) - .await - { - Err(err) if err.downcast_ref::().is_some() => { - tracing::error!( - "Schedule push zombie: {err}. Leaving flow job in queue for zombie detection to restart." - ); - Ok(JobOutcome::Completed) - } - other => { - other?; - Ok(JobOutcome::Completed) - } - } + .await?; + Ok(JobOutcome::Completed) } else { return Err(Error::internal_err( "Could not handle flow job with agent worker".to_string(), @@ -3881,15 +3892,16 @@ pub async fn handle_queued_job( #[cfg(not(feature = "enterprise"))] if let Connection::Sql(db) = conn { - if (job.concurrent_limit.is_some() - || windmill_common::runnable_settings::prefetch_cached_from_handle( - job.runnable_settings_handle, - db, - ) - .await? - .1 - .concurrent_limit - .is_some()) + if (windmill_queue::jobs::has_active_concurrency_limit(job.concurrent_limit) + || windmill_queue::jobs::has_active_concurrency_limit( + windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, + ) + .await? + .1 + .concurrent_limit, + )) && !job.kind.is_dependency() { logs.push_str("---\n"); @@ -4897,14 +4909,6 @@ pub async fn run_language_executor( .await; } } else if language == Some(ScriptLang::Bigquery) { - #[cfg(not(feature = "enterprise"))] - { - return Err(Error::ExecutionErr( - "Bigquery is only available with an enterprise license".to_string(), - )); - } - - #[allow(unreachable_code)] #[cfg(not(feature = "bigquery"))] { return Err(Error::internal_err( @@ -4912,7 +4916,7 @@ pub async fn run_language_executor( )); } - #[cfg(all(feature = "enterprise", feature = "bigquery"))] + #[cfg(feature = "bigquery")] { if run_inline { return Err(Error::internal_err( @@ -4934,14 +4938,14 @@ pub async fn run_language_executor( .await; } } else if language == Some(ScriptLang::Snowflake) { - #[cfg(not(feature = "enterprise"))] + #[cfg(not(feature = "snowflake"))] { - return Err(Error::ExecutionErr( - "Snowflake is only available with an enterprise license".to_string(), + return Err(Error::internal_err( + "Snowflake requires the snowflake feature to be enabled".to_string(), )); } - #[cfg(feature = "enterprise")] + #[cfg(feature = "snowflake")] { if run_inline { return Err(Error::internal_err( diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 10e8218a77..0793aff053 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -70,7 +70,7 @@ use windmill_common::{ use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - insert_concurrency_key, interpolate_args, + insert_concurrency_key_capped, interpolate_args, report_error_to_workspace_handler_or_critical_side_channel, try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, @@ -108,17 +108,6 @@ lazy_static::lazy_static! { quick_cache::sync::Cache::new(1024); } -#[derive(Debug)] -pub struct SchedulePushZombieError(pub String); - -impl std::fmt::Display for SchedulePushZombieError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::error::Error for SchedulePushZombieError {} - /// Helper function to write itered data to separate table /// Returns None if data was written to separate table, Some(itered) if it should be stored in JSONB async fn write_itered_to_db( @@ -1536,9 +1525,14 @@ pub async fn update_flow_status_after_job_completion_internal( let concurrency_key = tag_and_concurrency_key .as_ref() .and_then(|x| x.concurrency_key.clone()); - let concurrent_limit = tag_and_concurrency_key - .as_ref() - .and_then(|x| x.concurrent_limit); + // `concurrent_limit` here can come straight from the raw flow JSON (see + // get_tag_and_concurrency), bypassing the ConcurrencySettings deserialization guard, + // so a stored `0` must still be coerced to disabled before we register a key for it. + let concurrent_limit = windmill_common::runnable_settings::none_if_non_positive( + tag_and_concurrency_key + .as_ref() + .and_then(|x| x.concurrent_limit), + ); let concurrency_time_window_s = tag_and_concurrency_key .as_ref() .and_then(|x| x.concurrency_time_window_s); @@ -1563,12 +1557,13 @@ pub async fn update_flow_status_after_job_completion_internal( if concurrency_requires_args { let args = PushArgs::from(fetched_args.as_ref().unwrap()); if let Some(ck) = concurrency_key { - insert_concurrency_key( + insert_concurrency_key_capped( &flow_job.workspace_id, &args, &flow_job.runnable_path, JobKind::Flow, Some(ck), + concurrent_limit, db, flow, ) @@ -1578,12 +1573,13 @@ pub async fn update_flow_status_after_job_completion_internal( tag = Some(interpolate_args(t, &args, &flow_job.workspace_id)); } } else if concurrent_limit.is_some() { - insert_concurrency_key( + insert_concurrency_key_capped( &flow_job.workspace_id, &PushArgs::from(&HashMap::new()), &flow_job.runnable_path, JobKind::Flow, concurrency_key, + concurrent_limit, db, flow, ) @@ -2913,38 +2909,46 @@ pub async fn handle_flow( .sleep(tokio::time::sleep) .await; - // Non-retryable errors (QuotaExceeded, NotFound) are handled inside - // try_schedule_next_job (schedule disabled, returns None), so they never - // reach here. This handles only transient errors after retry exhaustion. if let Err(err) = schedule_push_result { - tracing::error!( - "Could not push next scheduled job for {} after retries: {err}. Disabling schedule.", - schedule.path - ); - if let Err(disable_err) = sqlx::query!( - "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", - err.to_string(), - &flow_job.workspace_id, - &schedule.path - ) - .execute(db) - .await - { + if matches!(err, Error::QuotaExceeded(_) | Error::NotFound(_)) { + // try_schedule_next_job disables on these, so reaching here means + // its own disable write failed. Retry it: rearm_schedule turns + // these into NoOp, so without disabling here the schedule would + // stay enabled yet never run. + if let Err(disable_err) = sqlx::query!( + "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3", + err.to_string(), + &flow_job.workspace_id, + &schedule.path + ) + .execute(db) + .await + { + report_error_to_workspace_handler_or_critical_side_channel( + &mini_job, + db, + format!( + "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + schedule.path, + ), + ) + .await; + } + } else { + // Transient error (DB contention, timeout) after retry exhaustion: + // not the schedule's fault. Report it but leave the schedule + // enabled; the current occurrence runs to completion and the + // unarmed-schedule reconciler re-arms the next one. Do not + // fail/requeue: a same-worker zombie would be canceled, losing it. report_error_to_workspace_handler_or_critical_side_channel( &mini_job, db, format!( - "Could not push next scheduled job for {} and could not disable schedule: {disable_err}", + "Could not push next scheduled job for {} after retries: {err}. Leaving it enabled for the unarmed-schedule reconciler to re-arm.", schedule.path, ), ) .await; - return Err(SchedulePushZombieError( - format!( - "Could not push or disable schedule {} after retries", - schedule.path - ), - ).into()); } } } else { @@ -3399,10 +3403,16 @@ async fn push_next_flow_job( // Persist approval user groups conditions, if any. Requires runnning the InputTransform let required_events = suspend.required_events.unwrap() as u16; let user_auth_required = suspend.user_auth_required.unwrap_or(false); - if user_auth_required { - let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + let self_approval_disabled = suspend.self_approval_disabled.unwrap_or(false); + // self_approval_disabled must be persisted even without user_auth_required, otherwise + // the resume boundary sees no approval_conditions and the restriction is silently + // dropped. user_groups_required only applies together with user_auth_required. + if user_auth_required || self_approval_disabled { let user_groups_required: Vec; - if let Some(user_groups_required_as_input_transform) = suspend.user_groups_required + if !user_auth_required { + user_groups_required = Vec::new(); + } else if let Some(user_groups_required_as_input_transform) = + suspend.user_groups_required { match user_groups_required_as_input_transform { InputTransform::Static { value } => { @@ -3567,7 +3577,7 @@ async fn push_next_flow_job( count: required_events, job: last }), - (required_events - resume_messages.len() as u16) as i32, + (required_events.saturating_sub(resume_messages.len() as u16)) as i32, Duration::from_secs( suspend.timeout.map(|t| t.into()).unwrap_or_else(|| 30 * 60) ) as Duration, @@ -4384,13 +4394,10 @@ async fn push_next_flow_job( ) .await?; - if timeout_value < 0 { - return Err(Error::ExecutionErr( - "Timeout value cannot be negative".to_string(), - )); - } - - Some(timeout_value) + // A `<= 0` step timeout (including a negative eval) means "no override": fall back + // to the referenced runnable's own timeout rather than a 0-second/negative timeout + // that would kill the step instantly. + effective_flow_step_timeout(Some(timeout_value), payload_tag.timeout) } else { payload_tag.timeout }; @@ -6043,6 +6050,18 @@ async fn flow_to_payload( }) } +/// Effective timeout for a flow step given the module's (already-evaluated) timeout override and +/// the timeout inherited from the referenced runnable. A `<= 0` override — or none — means "no +/// override": fall back to the inherited value (which is itself `None` when unset, i.e. the +/// instance default). A positive override wins. This keeps a step `timeout: 0` equivalent to an +/// omitted one rather than a 0-second, instant-kill timeout. +pub(crate) fn effective_flow_step_timeout( + module_override: Option, + inherited: Option, +) -> Option { + windmill_common::runnable_settings::none_if_non_positive(module_override).or(inherited) +} + pub async fn script_to_payload( script_hash: Option, script_path: String, @@ -6132,11 +6151,13 @@ pub async fn script_to_payload( module.delete_after_use.unwrap_or(false) || delete_after_use.unwrap_or(false); let final_delete_after_secs = module.delete_after_secs.or(delete_after_secs); - let flow_step_timeout = if module.timeout.is_some() { - None - } else { - script_timeout - }; + // Always carry the referenced script's own timeout as the inherited fallback. The module's + // timeout override (if any) is selected at the push site, where a `<= 0` override is treated + // as "no override" and falls back to this value — so `timeout: 0` on a step means "use the + // script's timeout", not a 0-second (immediate-kill) timeout. Normalize the inherited value + // too, so a legacy `0` script timeout resolves to the default rather than a zero-second kill. + let flow_step_timeout = + windmill_common::runnable_settings::none_if_non_positive(script_timeout); Ok(JobPayloadWithTag { payload, tag, @@ -6261,9 +6282,25 @@ pub async fn get_previous_job_result( #[cfg(test)] mod tests { - use super::extract_chat_message_from_flow_result; + use super::{effective_flow_step_timeout, extract_chat_message_from_flow_result}; use serde_json::{json, value::to_raw_value}; + // A `<= 0` step timeout override must behave as "no override" and inherit the referenced + // script's timeout, not collapse to a 0-second (instant-kill) timeout. A positive override + // still wins. Guards the flow-step timeout footgun. + #[test] + fn flow_step_timeout_zero_or_negative_inherits_script_timeout() { + // zero / negative override -> inherited script timeout + assert_eq!(effective_flow_step_timeout(Some(0), Some(300)), Some(300)); + assert_eq!(effective_flow_step_timeout(Some(-5), Some(300)), Some(300)); + // no inherited timeout either -> None (falls through to the instance default) + assert_eq!(effective_flow_step_timeout(Some(0), None), None); + // positive override wins over the inherited value + assert_eq!(effective_flow_step_timeout(Some(120), Some(300)), Some(120)); + // no override -> inherited + assert_eq!(effective_flow_step_timeout(None, Some(300)), Some(300)); + } + #[test] fn pretty_prints_full_result_when_no_override_is_present() { let value = json!({ diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index d98a2cd2e0..5c695ac64f 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -2468,7 +2468,7 @@ async fn ansible_dep( let ansible_lockfile; - create_ansible_cfg(Some(&reqs), job_dir, false)?; + create_ansible_cfg(Some(&reqs), job_dir, false, job_id)?; if let Some(collections) = reqs.roles_and_collections.as_ref() { install_galaxy_collections( diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index b325f3cf08..a36ad4c348 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -346,6 +346,34 @@ pub async fn ping_job_status( } } +/// Keeps the job's ping fresh during phases that run before the executor's own polling +/// loop starts (volume setup, s3object materialization, ...). Without it, a slow wait or +/// download with no ping in between can exceed ZOMBIE_JOB_TIMEOUT (default 60s) and get +/// the job falsely restarted as a zombie. +pub(crate) struct JobPingHeartbeat(tokio::task::JoinHandle<()>); + +impl JobPingHeartbeat { + pub(crate) fn start(conn: &Connection, job_id: Uuid, context: &'static str) -> Self { + let conn = conn.clone(); + JobPingHeartbeat(tokio::spawn(async move { + // 10s stays well under the 60s zombie timeout + let mut interval = tokio::time::interval(std::time::Duration::from_secs(10)); + loop { + interval.tick().await; + if let Err(e) = ping_job_status(&conn, &job_id, None, None).await { + tracing::warn!("failed to ping job {job_id} during {context}: {e}"); + } + } + })) + } +} + +impl Drop for JobPingHeartbeat { + fn drop(&mut self) { + self.0.abort(); + } +} + pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: &str) { match conn { Connection::Sql(db) => { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 1cc713ae60..debc81ac94 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.759.0"; +export const VERSION = "v1.770.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bun.lock b/cli/bun.lock index a745acbcba..c783e411c7 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -19,16 +19,16 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", - "windmill-parser-wasm-asset": "1.749.0", + "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.693.1", "windmill-parser-wasm-py-imports": "1.693.1", "windmill-parser-wasm-r": "1.668.1", - "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", @@ -290,11 +290,11 @@ "utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="], - "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.749.0", "", {}, "sha512-gj8g9sWQ0tXKfXso7xJxR56sS8Loe/RsnFy+0af5R8siZeCag9ikquGbQ8d8kOqIK2U8eCNA0LqsU/xAFDJIOg=="], + "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.753.0", "", {}, "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="], "windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="], - "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.510.1", "", {}, "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ=="], + "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.761.0", "", {}, "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q=="], "windmill-parser-wasm-java": ["windmill-parser-wasm-java@1.510.1", "", {}, "sha512-Zle+JZT/ZwUArUVacUudYlS+CaHp2lSnkqD/IhWaRUG+gcv26VbERnrrHPonqXbVMS+eA9ElfXrFM5j0ukaXUw=="], @@ -308,7 +308,7 @@ "windmill-parser-wasm-r": ["windmill-parser-wasm-r@1.668.1", "", {}, "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ=="], - "windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.692.0", "", {}, "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw=="], + "windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.764.0", "", {}, "sha512-V2eFdKD90gqWikOvjl2fwMpFqiFt/21+4iQMbiNJYl7Lm2UiEcEZ4r9bpgJLG4TLOLqvD6+u4Ju3WaytxN2O2w=="], "windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="], diff --git a/cli/package-lock.json b/cli/package-lock.json index a9d72696b2..b0b610a045 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -20,16 +20,16 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", - "windmill-parser-wasm-asset": "1.749.0", + "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.693.1", "windmill-parser-wasm-py-imports": "1.693.1", "windmill-parser-wasm-r": "1.668.1", - "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", @@ -793,6 +793,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1412,9 +1413,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.749.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.749.0.tgz", - "integrity": "sha512-gj8g9sWQ0tXKfXso7xJxR56sS8Loe/RsnFy+0af5R8siZeCag9ikquGbQ8d8kOqIK2U8eCNA0LqsU/xAFDJIOg==" + "version": "1.753.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.753.0.tgz", + "integrity": "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", @@ -1422,9 +1423,9 @@ "integrity": "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ==" }, "node_modules/windmill-parser-wasm-go": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.510.1.tgz", - "integrity": "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ==" + "version": "1.761.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.761.0.tgz", + "integrity": "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q==" }, "node_modules/windmill-parser-wasm-java": { "version": "1.510.1", @@ -1457,9 +1458,9 @@ "integrity": "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.692.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.692.0.tgz", - "integrity": "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw==" + "version": "1.764.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.764.0.tgz", + "integrity": "sha512-V2eFdKD90gqWikOvjl2fwMpFqiFt/21+4iQMbiNJYl7Lm2UiEcEZ4r9bpgJLG4TLOLqvD6+u4Ju3WaytxN2O2w==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/cli/package.json b/cli/package.json index 589bc52271..548588a402 100644 --- a/cli/package.json +++ b/cli/package.json @@ -28,16 +28,16 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", - "windmill-parser-wasm-asset": "1.749.0", + "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.693.1", "windmill-parser-wasm-py-imports": "1.693.1", "windmill-parser-wasm-r": "1.668.1", - "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", @@ -54,4 +54,4 @@ "@types/ws": "^8.5.0", "typescript": "^5.7.0" } -} +} \ No newline at end of file diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 07ee266145..c515e1e86e 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -35,10 +35,24 @@ export const DEFAULT_BUILD_OPTIONS = { loader: { ".css": "css" as const, }, + // esbuild export conditions safe for any app: "style" resolves tailwindcss v4's CSS + // entry (@import "tailwindcss"); "module" is re-added because esbuild drops its + // auto-included "module" default once any custom condition is set. The Svelte-only + // "svelte" condition is gated per-app in conditionsFor(). + conditions: ["style", "module"], logLevel: "info" as const, write: true, }; +// "svelte" points at raw .svelte sources that only compile with the Svelte plugin, so +// enable it only for Svelte apps — for a plain app a Svelte-dual-published import would +// otherwise resolve to .svelte and hard-fail with no loader configured. +function conditionsFor(svelte: boolean): string[] { + return svelte + ? [...DEFAULT_BUILD_OPTIONS.conditions, "svelte"] + : DEFAULT_BUILD_OPTIONS.conditions; +} + /** * Detects which frontend frameworks are present in package.json */ @@ -284,6 +298,7 @@ export async function createBundle( const buildOptions = { ...DEFAULT_BUILD_OPTIONS, + conditions: conditionsFor(frameworks.svelte), entryPoints: [entryPoint], outfile, sourcemap, @@ -337,11 +352,13 @@ export async function createBundle( /** * Gets the esbuild build options for use in watch mode (dev server) * @param entryPoint Entry point file + * @param svelte Whether the app is a Svelte app (enables the "svelte" condition) * @returns esbuild build options */ -export function getDevBuildOptions(entryPoint: string = "index.tsx") { +export function getDevBuildOptions(entryPoint: string = "index.tsx", svelte = false) { return { ...DEFAULT_BUILD_OPTIONS, + conditions: conditionsFor(svelte), entryPoints: [entryPoint], outfile: "dist/bundle.js", sourcemap: true, diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 1a226ec5ef..1842e1d007 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -538,7 +538,7 @@ async function dev(opts: DevOptions, appFolder?: string) { }); } - const buildOptions = getDevBuildOptions(entryPoint); + const buildOptions = getDevBuildOptions(entryPoint, frameworks.svelte); // Load framework-specific plugins (svelte, vue) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); diff --git a/cli/src/commands/gitsync-settings/gitsync-settings.ts b/cli/src/commands/gitsync-settings/gitsync-settings.ts index 5c27e032b5..efaf00dcb5 100644 --- a/cli/src/commands/gitsync-settings/gitsync-settings.ts +++ b/cli/src/commands/gitsync-settings/gitsync-settings.ts @@ -1,6 +1,7 @@ import { Command } from "@cliffy/command"; import { pullGitSyncSettings } from "./pull.ts"; import { pushGitSyncSettings } from "./push.ts"; +import { gitSyncStatus } from "./status.ts"; const command = new Command() .description( @@ -54,7 +55,13 @@ const command = new Command() "--promotion ", "Use promotionOverrides from the specified branch instead of regular overrides" ) - .action(pushGitSyncSettings as any); + .action(pushGitSyncSettings as any) + .command("status") + .description( + "Report how local changes deploy to the workspace (git push vs wmill sync push)", + ) + .option("--json-output", "Output in JSON format") + .action(gitSyncStatus as any); -export { pullGitSyncSettings, pushGitSyncSettings }; +export { pullGitSyncSettings, pushGitSyncSettings, gitSyncStatus }; export default command; diff --git a/cli/src/commands/gitsync-settings/status.ts b/cli/src/commands/gitsync-settings/status.ts new file mode 100644 index 0000000000..23ff2a9f0a --- /dev/null +++ b/cli/src/commands/gitsync-settings/status.ts @@ -0,0 +1,57 @@ +import { colors } from "@cliffy/ansi/colors"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { getCurrentGitBranch, isGitRepository } from "../../utils/git.ts"; + +export async function gitSyncStatus(opts: GlobalOptions & { jsonOutput?: boolean }) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const branch = isGitRepository() ? getCurrentGitBranch() : null; + + // The backend reports whether exactly one licensed, deliverable auto-pull repo + // tracks this branch — in which case a `git push` deploys. It does not check + // the remote URL: with a single synced repo the checkout is unambiguously it, + // and with several the backend returns false so we defer to the user here. + const mode = await wmill.getGitSyncDeployMode({ + workspace: workspace.workspaceId, + branch: branch ?? undefined, + }); + + const deployCommand = mode.deploy_on_push ? "git push" : null; + + if (opts.jsonOutput) { + // console.log (not log.info, which wraps in ANSI color) so the output pipes cleanly to jq. + console.log( + JSON.stringify( + { ...mode, current_branch: branch, deploy_command: deployCommand }, + null, + 2, + ), + ); + return; + } + + if (mode.deploy_on_push) { + log.info( + colors.green( + `Git-sync auto-pull deploys this branch: pushing '${branch}' deploys to the workspace.`, + ), + ); + log.info(`Recommended deploy command: ${colors.bold("git push")}`); + } else { + log.info( + colors.yellow( + mode.configured + ? "Couldn't confirm a backend auto-pull deploy for this checkout (no single repo unambiguously tracks this branch)." + : "No git-sync repository is configured for this workspace on the backend.", + ), + ); + log.info( + "This is not confirmed deploy-on-push, not a definite no. Ask the user how this repo deploys: `git push` (a CI workflow, or a git-sync repo that couldn't be disambiguated) or `wmill sync push`. Record the answer as a `Deploy mode:` line in AGENTS.md so later sessions skip the question (see the Deploying section).", + ); + } +} diff --git a/cli/src/commands/pipeline/boundedCascade.ts b/cli/src/commands/pipeline/boundedCascade.ts index 69166726c5..31c780a272 100644 --- a/cli/src/commands/pipeline/boundedCascade.ts +++ b/cli/src/commands/pipeline/boundedCascade.ts @@ -71,6 +71,7 @@ const assetNodeId = (kind: string, path: string): string => `${kind}:${path}`; const NON_AUTORUN_TRIGGER_KINDS = new Set([ "kafka", "mqtt", + "amqp", "nats", "postgres", "sqs", @@ -86,12 +87,10 @@ export function assetUriToNodeId(uri: string): string | undefined { if (!m) return undefined; const prefix = m[1].toLowerCase(); const kind = prefix === "s3" ? "s3object" : prefix; - // Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys so a - // `--to s3:///exports/x` token resolves to the canonical graph node - // `s3object:exports/x` (default storage), same as `s3://exports/x`, and a - // canonical key never starts with `/`. - const path = kind === "s3object" ? m[2].replace(/^\/+/, "") : m[2]; - return `${kind}:${path}`; + // The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3 + // path encodes the storage, with a leading `/` for the workspace default + // (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`. + return `${kind}:${m[2]}`; } export type LineageDag = { diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index eb3350c818..bb51cd98eb 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -259,6 +259,7 @@ const NATIVE_KINDS = new Set([ "email", "kafka", "mqtt", + "amqp", "nats", "postgres", "sqs", @@ -299,13 +300,10 @@ function fallbackParse(content: string, language: string): ParseAssetsRaw { if (uri) { const prefix = uri[1].toLowerCase(); const kind = prefix === "s3" ? "s3object" : prefix; - // Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys - // so `s3:///key` (default storage) and `s3://key` / DuckDB canonicalize - // to the same node id (and a canonical key never starts with `/`) — - // otherwise a go/bash fallback consumer's `// on s3:///x` would not - // connect to a wasm-inferred `x` producer. - const path = kind === "s3object" ? uri[2].replace(/^\/+/, "") : uri[2]; - out.triggers!.push({ kind: "asset", asset_kind: kind, path }); + // The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3 + // path encodes the storage, with a leading `/` for the workspace default + // (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`. + out.triggers!.push({ kind: "asset", asset_kind: kind, path: uri[2] }); } else if (NATIVE_KINDS.has(firstTok) && rest === firstTok) { // A native marker (`// on data_upload`) must stand alone: the canonical // parser rejects a marker line with trailing content (`// on data_upload @@ -392,14 +390,10 @@ export function parseMuteAnnotations(content: string): { } for (const [prefix, kind] of MUTE_ASSET_PREFIXES) { if (arg.startsWith(prefix)) { - // S3 canonicalization as in `parse_asset_syntax`: strip every leading - // slash so `s3:///key` (default storage) mutes the same node as the - // inferred bare `key`. - const p = - kind === "s3object" - ? arg.slice(prefix.length).replace(/^\/+/, "") - : arg.slice(prefix.length); - muted.add(`${kind}:${p}`); + // The suffix is kept verbatim, as in `parse_asset_syntax` — a muted + // `s3:///key` (default storage, path `/key`) only matches an inferred + // default-storage read of the same object. + muted.add(`${kind}:${arg.slice(prefix.length)}`); break; } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 3db9c7b53e..82fe63719b 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -100,6 +100,30 @@ export function isRawAppBackendPath(filePath: string): boolean { return isRawAppBackendPathInternal(filePath); } +/** + * The positive-only runnable settings (concurrent_limit, timeout, ...) treat any `<= 0` + * value as "unset": the backend coerces it to null (a 0-slot concurrency limit bricks the + * runnable, a 0s timeout kills every run). Coerce to undefined so it is serialized as + * omitted, never as 0, and redeploys don't churn against the backend-normalized value. + */ +export function nonePositiveInt( + v: number | undefined | null +): number | undefined { + return v != null && v > 0 ? v : undefined; +} + +/** + * Normalize a concurrent_limit + its time window together: when the limit is disabled + * (<= 0) the window is dropped too. Returns [concurrent_limit, concurrency_time_window_s]. + */ +export function normalizeConcurrency( + concurrentLimit: number | undefined | null, + concurrencyTimeWindowS?: number | undefined | null +): [number | undefined, number | undefined] { + const limit = nonePositiveInt(concurrentLimit); + return limit === undefined ? [undefined, undefined] : [limit, concurrencyTimeWindowS ?? undefined]; +} + /** * Checks if a path is inside a normal app folder (inline script). * Matches patterns like: .../myApp.app/... or .../myApp__app/... @@ -469,6 +493,15 @@ export async function handleFile( const moduleFolderPath = scriptBasePath + getModuleFolderSuffix(); const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint); + // A concurrent_limit of <= 0 means "concurrency disabled", not "zero slots" (which + // would brick the runnable at the queue's concurrency gate). Emit it as omitted rather + // than 0 so a redeploy never re-persists a zero-slot limit, and drop the now-meaningless + // time window alongside it. Mirrors the backend's ConcurrencySettings::normalized. + const [normConcurrentLimit, normConcurrencyTimeWindowS] = normalizeConcurrency( + typed?.concurrent_limit, + typed?.concurrency_time_window_s + ); + const requestBodyCommon: NewScript = { content, description: typed?.description ?? "", @@ -482,8 +515,8 @@ export async function handleFile( ws_error_handler_muted: typed?.ws_error_handler_muted, dedicated_worker: typed?.dedicated_worker, cache_ttl: typed?.cache_ttl, - concurrency_time_window_s: typed?.concurrency_time_window_s, - concurrent_limit: typed?.concurrent_limit, + concurrency_time_window_s: normConcurrencyTimeWindowS, + concurrent_limit: normConcurrentLimit, deployment_message: message, restart_unless_cancelled: typed?.restart_unless_cancelled, visible_to_runner_only: typed?.visible_to_runner_only, @@ -493,7 +526,7 @@ export async function handleFile( debounce_key: typed?.debounce_key, debounce_delay_s: typed?.debounce_delay_s, codebase: await codebase?.getDigest(forceTar), - timeout: typed?.timeout, + timeout: nonePositiveInt(typed?.timeout), on_behalf_of_email: typed?.on_behalf_of_email, envs: typed?.envs, modules: modules, @@ -530,9 +563,13 @@ export async function handleFile( remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && - typed.concurrency_time_window_s == - remote.concurrency_time_window_s && - typed.concurrent_limit == remote.concurrent_limit && + normConcurrencyTimeWindowS == + normalizeConcurrency( + remote.concurrent_limit, + remote.concurrency_time_window_s + )[1] && + normConcurrentLimit == + normalizeConcurrency(remote.concurrent_limit)[0] && Boolean(typed.restart_unless_cancelled) == Boolean(remote.restart_unless_cancelled) && Boolean(typed.visible_to_runner_only) == @@ -540,7 +577,7 @@ export async function handleFile( Boolean(typed.has_preprocessor) == Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && - typed.timeout == remote.timeout && + nonePositiveInt(typed.timeout) == nonePositiveInt(remote.timeout) && //@ts-ignore typed.concurrency_key == remote["concurrency_key"] && typed.debounce_key == remote["debounce_key"] && diff --git a/cli/src/commands/shared_ui.ts b/cli/src/commands/shared_ui.ts index e3f719d77f..3c01870fd1 100644 --- a/cli/src/commands/shared_ui.ts +++ b/cli/src/commands/shared_ui.ts @@ -26,34 +26,77 @@ async function readDirRecursive( return out; } +export type SharedUiChange = + | { type: "added"; path: string } + | { type: "edited"; path: string; before: string; after: string } + | { type: "deleted"; path: string }; + +/** + * Diff the local /ui/ folder against the workspace's shared UI store in + * the push direction (local -> remote), returning entries whose `path` is + * prefixed with `ui/`. This is the same comparison pushSharedUi applies, so the + * dry-run preview and the real push never diverge. + * + * Mirrors pushSharedUi's no-op: with no local ui/ folder there is nothing to + * push, so the apply is a no-op and the preview must be empty (even when the + * remote store is non-empty) to avoid phantom diffs the apply won't perform. + */ +export async function diffSharedUi(workspace: string): Promise { + const localDir = path.join(process.cwd(), SHARED_UI_DIR); + if (!fs.existsSync(localDir)) { + return []; + } + const files = await readDirRecursive(localDir); + + let remote: Record = {}; + try { + const got = await wmill.getSharedUi({ workspace }); + remote = got.files ?? {}; + } catch { + // If endpoint missing or unauthorized, treat remote as empty (the push + // would attempt the PUT anyway). + } + + // Use Object.hasOwn, not `in`: a file named after an Object.prototype member + // (e.g. ui/toString) would otherwise register as always-present and be + // misdiffed. + const changes: SharedUiChange[] = []; + for (const [rel, content] of Object.entries(files)) { + const p = `${SHARED_UI_DIR}/${rel}`; + if (!Object.hasOwn(remote, rel)) { + changes.push({ type: "added", path: p }); + } else if (remote[rel] !== content) { + changes.push({ type: "edited", path: p, before: remote[rel], after: content }); + } + } + for (const rel of Object.keys(remote)) { + if (!Object.hasOwn(files, rel)) { + changes.push({ type: "deleted", path: `${SHARED_UI_DIR}/${rel}` }); + } + } + return changes; +} + /** * Push the local /ui/ folder to the workspace's shared UI store. - * Returns true if a push was performed, false if the folder is missing or empty. + * Returns true if a push was performed, false if the folder is missing or + * already matches the remote store. Note an empty-but-existing folder still + * pushes an empty map (clearing the remote store) if the remote is non-empty. */ export async function pushSharedUi(workspace: string): Promise { const localDir = path.join(process.cwd(), SHARED_UI_DIR); if (!fs.existsSync(localDir)) { return false; } - const files = await readDirRecursive(localDir); - // Skip if no change - let remote: Record = {}; - try { - const got = await wmill.getSharedUi({ workspace }); - remote = got.files ?? {}; - } catch { - // If endpoint missing or unauthorized, just attempt the PUT - } - - if ( - Object.keys(remote).length === Object.keys(files).length && - Object.entries(files).every(([k, v]) => remote[k] === v) - ) { + // Skip if no change — reuse diffSharedUi so preview and push never diverge. + const diff = await diffSharedUi(workspace); + if (diff.length === 0) { log.info(colors.gray("Shared UI folder up to date")); return false; } + const files = await readDirRecursive(localDir); await wmill.updateSharedUi({ workspace, requestBody: { files }, diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 65ce10eaad..f10195670a 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -28,7 +28,7 @@ import { } from "../../types.ts"; import { downloadZip } from "./pull.ts"; import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; -import { pullSharedUi, pushSharedUi } from "../shared_ui.ts"; +import { diffSharedUi, pullSharedUi, pushSharedUi } from "../shared_ui.ts"; import { pushMigrationFromDisk, offerToRunNewMigrations, @@ -1008,7 +1008,11 @@ function ZipFSElement( } if (stripOnBehalfOf) { - (flow as any).has_on_behalf_of = !!(flow as any).on_behalf_of_email; + // Only emit the flag when set; a `false` here is the default and + // would produce a spurious diff for every ownerless flow. + if ((flow as any).on_behalf_of_email) { + (flow as any).has_on_behalf_of = true; + } delete (flow as any).on_behalf_of_email; } @@ -1293,7 +1297,11 @@ function ZipFSElement( parsed["codebase"] = undefined; } if (stripOnBehalfOf) { - parsed["has_on_behalf_of"] = !!parsed["on_behalf_of_email"]; + // Only emit the flag when set; a `false` here is the default and + // would produce a spurious diff for every ownerless script. + if (parsed["on_behalf_of_email"]) { + parsed["has_on_behalf_of"] = true; + } delete parsed["on_behalf_of_email"]; } // Modules are stored as files in __mod/ folder, not in metadata @@ -1340,13 +1348,19 @@ function ZipFSElement( if (stripOnBehalfOf) { const isSchedule = p.endsWith(".schedule.json"); const isTrigger = p.endsWith("_trigger.json"); + // Only emit the flag when set; a `false` here is the default and + // would produce a spurious diff for every ownerless schedule/trigger. if (isSchedule) { - parsed["has_permissioned_as"] = !!parsed["permissioned_as"]; + if (parsed["permissioned_as"]) { + parsed["has_permissioned_as"] = true; + } delete parsed["permissioned_as"]; delete parsed["email"]; delete parsed["edited_by"]; } else if (isTrigger) { - parsed["has_permissioned_as"] = !!parsed["permissioned_as"]; + if (parsed["permissioned_as"]) { + parsed["has_permissioned_as"] = true; + } delete parsed["permissioned_as"]; delete parsed["edited_by"]; } @@ -1703,6 +1717,7 @@ export async function elementsToMap( path.endsWith(".nats_trigger" + ext) || path.endsWith(".postgres_trigger" + ext) || path.endsWith(".mqtt_trigger" + ext) || + path.endsWith(".amqp_trigger" + ext) || path.endsWith(".sqs_trigger" + ext) || path.endsWith(".gcp_trigger" + ext) || path.endsWith(".azure_trigger" + ext) || @@ -2456,6 +2471,7 @@ function getOrderFromPath(p: string) { typ == "nats_trigger" || typ == "postgres_trigger" || typ == "mqtt_trigger" || + typ == "amqp_trigger" || typ == "sqs_trigger" || typ == "gcp_trigger" || typ == "azure_trigger" || @@ -2779,6 +2795,8 @@ export async function pull( gitDeployItems?: string; onlyCreateBranch?: boolean; parentWorkspaceId?: string; + devWorkspaceLabel?: string; + parentDevWorkspaceLabel?: string; gitCommitterEmail?: string; gitCommitterName?: string; }, @@ -2847,37 +2865,58 @@ export async function pull( } const clonedBranchName = getCurrentGitBranch() ?? "main"; - // Fork workspaces force-disable use_individual_branch / group_by_folder - // (1:1 with the hub script's inner()). - const targetIsFork = isForkWorkspace(workspace.workspaceId); - const useIndividualBranch = targetIsFork + // Throwaway forks force-disable use_individual_branch / group_by_folder + // (1:1 with the hub script's inner()). A dev workspace is the exception: it + // honors promotion mode and gets per-item wm_deploy/** branches. Dev + // workspaces have a prefix-less id, so detect them via the environment label + // the backend passes with the deploy. + const targetIsFork = isForkWorkspace( + workspace.workspaceId, + opts.parentWorkspaceId, + ); + const forceOffPromotion = targetIsFork && !opts.devWorkspaceLabel; + const useIndividualBranch = forceOffPromotion ? false : !!opts.useIndividualBranch; - const groupByFolder = targetIsFork ? false : !!opts.groupByFolder; + const groupByFolder = forceOffPromotion ? false : !!opts.groupByFolder; - // Fork-of-a-fork: only when the parent workspace is itself a fork, root - // the new branch on the parent's fork branch (mirrors the hub script's - // `parent_workspace_id?.startsWith(FORKED_…)` gate). - if (opts.parentWorkspaceId && isForkWorkspace(opts.parentWorkspaceId)) { - const parentBranch = computeGitSyncDeployBranch({ - workspaceId: opts.parentWorkspaceId, - items: deployItems, - useIndividualBranch, - groupByFolder, - clonedBranchName, - }); - if (parentBranch && parentBranch !== clonedBranchName) { - checkoutGitSyncDeployBranch(parentBranch); - } + // Fork-of-a-fork: when the parent workspace is itself a fork, root the new + // branch on the parent's fork branch (the content this fork diverged from). + // A dev-workspace parent has a prefix-less id the prefix check can't see, so + // the backend passes its environment label; its branch is the label verbatim. + const parentBranch = opts.parentDevWorkspaceLabel + ? opts.parentDevWorkspaceLabel + : opts.parentWorkspaceId && isForkWorkspace(opts.parentWorkspaceId) + ? computeGitSyncDeployBranch({ + workspaceId: opts.parentWorkspaceId, + items: deployItems, + useIndividualBranch, + groupByFolder, + clonedBranchName, + }) + : null; + if (parentBranch && parentBranch !== clonedBranchName) { + checkoutGitSyncDeployBranch(parentBranch); } const deployBranch = computeGitSyncDeployBranch({ workspaceId: workspace.workspaceId, + parentWorkspaceId: opts.parentWorkspaceId, + devWorkspaceLabel: opts.devWorkspaceLabel, items: deployItems, useIndividualBranch, groupByFolder, clonedBranchName, }); + // A dev workspace whose environment-label branch equals the repository's + // tracked branch would silently commit the fork's content straight to the + // tracked branch. Refuse instead of deploying in place. + if (targetIsFork && deployBranch && deployBranch === clonedBranchName) { + log.error( + `Fork branch '${deployBranch}' equals the checked-out branch '${clonedBranchName}'; refusing to deploy a fork directly to the tracked branch. Use a different dev workspace label or tracked branch.`, + ); + process.exit(1); + } if (deployBranch && deployBranch !== clonedBranchName) { checkoutGitSyncDeployBranch(deployBranch); } @@ -3394,6 +3433,8 @@ export async function gitDeploy( groupByFolder?: boolean; onlyCreateBranch?: boolean; parentWorkspaceId?: string; + devWorkspaceLabel?: string; + parentDevWorkspaceLabel?: string; skipSecrets?: boolean; gitCommitterEmail?: string; gitCommitterName?: string; @@ -3409,12 +3450,14 @@ export async function gitDeploy( } } - // Fork workspaces force-disable use_individual_branch / group_by_folder - // (1:1 with the hub script's inner()): a fork always syncs to its own - // wm-fork// branch, and — critically — that disabling also - // flips the include/promotion derivation below. - const isFork = isForkWorkspace(opts.workspace ?? ""); - const useIndividualBranch = isFork ? false : !!opts.useIndividualBranch; + // Throwaway forks force-disable use_individual_branch / group_by_folder (1:1 + // with the hub script's inner()): they always sync to their own + // wm-fork// branch, and — critically — that disabling also flips + // the include/promotion derivation below. A dev workspace is the exception: it + // honors promotion mode, detected via the environment label the backend passes. + const isFork = isForkWorkspace(opts.workspace ?? "", opts.parentWorkspaceId); + const useIndividualBranch = + isFork && !opts.devWorkspaceLabel ? false : !!opts.useIndividualBranch; // Derive the include filters from the deployed items (replaces the hub // script's regexFromPath + per-kind --include-* construction). @@ -3451,6 +3494,9 @@ export async function gitDeploy( // are self-describing via their `migrations/datatable/...` path, so they get no // label prefix. function changeTypeLabel(p: string): string { + // Shared UI files (ui/…) are not wmill items — getTypeStrFromPath throws on + // them (e.g. ui/config.json). Label them directly. + if (p === "ui" || p.startsWith("ui/")) return "shared UI "; const t = getTypeStrFromPath(p); return t === "datatable_migration" ? "" : `${t} `; } @@ -3500,7 +3546,6 @@ function prettyChanges( ), ); } else if (change.name === "edited") { - const changeType = getTypeStrFromPath(change.path); log.info( colors.yellow( `~ ${changeTypeLabel(change.path)}` + @@ -3510,6 +3555,12 @@ function prettyChanges( ), ); if (change.before != change.after) { + // Shared UI files (ui/…) aren't wmill items; getTypeStrFromPath throws + // on them, so fall back to a plain diff. + const changeType = + change.path === "ui" || change.path.startsWith("ui/") + ? "shared_ui" + : getTypeStrFromPath(change.path); if (changeType === "encryption_key") { showDiff( redactEncryptionKey(change.before), @@ -4085,6 +4136,33 @@ export async function push( await fetchRemoteVersion(workspace); + // Shared UI (the ui/ folder) is pushed out-of-band via pushSharedUi on apply + // and is excluded from the file diff (isNotWmillFile), so surface its diff in + // the dry-run preview. Without this the "Pull from repo" preview reads "no + // changes" even when the apply will overwrite the shared-UI store. Folded in + // only for dry-run (before the count/summary below) so the apply path is + // unchanged (pushSharedUi still runs) and the summary count includes ui/. + if (opts.dryRun) { + try { + for (const c of await diffSharedUi(workspace.workspaceId)) { + if (c.type === "added") { + changes.push({ name: "added", path: c.path, content: "" }); + } else if (c.type === "deleted") { + changes.push({ name: "deleted", path: c.path }); + } else { + changes.push({ + name: "edited", + path: c.path, + before: c.before, + after: c.after, + }); + } + } + } catch (e) { + log.warn(`Failed to compute shared UI diff for dry-run preview: ${e}`); + } + } + log.info( `remote (${workspace.name}) <- local: ${changes.length} changes to apply`, ); @@ -4200,7 +4278,7 @@ export async function push( } } const rules = folderRulesCache.get(folderName)!; - const remotePath = change.path.replace(/\.(script|schedule|http_trigger|websocket_trigger|kafka_trigger|nats_trigger|postgres_trigger|mqtt_trigger|sqs_trigger|gcp_trigger|azure_trigger|email_trigger)\.(yaml|json)$/, "").replace(/(\.flow|__flow)\/flow\.(yaml|json)$/, "").replace(/\.(app|raw_app)(\/app\.(yaml|json))?$/, ""); + const remotePath = change.path.replace(/\.(script|schedule|http_trigger|websocket_trigger|kafka_trigger|nats_trigger|postgres_trigger|mqtt_trigger|amqp_trigger|sqs_trigger|gcp_trigger|azure_trigger|email_trigger)\.(yaml|json)$/, "").replace(/(\.flow|__flow)\/flow\.(yaml|json)$/, "").replace(/\.(app|raw_app)(\/app\.(yaml|json))?$/, ""); const relative = remotePath.slice(`f/${folderName}/`.length); if (!relative) continue; for (const rule of rules) { @@ -4894,6 +4972,12 @@ export async function push( path: removeSuffix(target, ".mqtt_trigger.json"), }); break; + case "amqp_trigger": + await wmill.deleteAmqpTrigger({ + workspace: workspaceId, + path: removeSuffix(target, ".amqp_trigger.json"), + }); + break; case "sqs_trigger": await wmill.deleteSqsTrigger({ workspace: workspaceId, @@ -5121,16 +5205,29 @@ export async function push( ); } } else { - try { - await pushSharedUi(workspace.workspaceId); - } catch (e) { - log.warn(`Failed to push shared UI folder: ${e}`); + // Dry-run with no changes reaches here (a ui/ diff would have made changes + // non-empty and returned above); never mutate the remote in that case. + let sharedUiPushed = false; + if (!opts.dryRun) { + try { + sharedUiPushed = await pushSharedUi(workspace.workspaceId); + } catch (e) { + log.warn(`Failed to push shared UI folder: ${e}`); + } } // No changes pushed, so no new datatable migrations to run. if (opts.jsonOutput) { + // Shared UI is out-of-band from the file diff (total counts diffed + // files), but don't claim "No changes" when the ui/ store was written. console.log( JSON.stringify( - { success: true, message: "No changes to push", total: 0 }, + { + success: true, + message: sharedUiPushed + ? "Pushed shared UI changes" + : "No changes to push", + total: 0, + }, null, 2, ), @@ -5304,6 +5401,14 @@ const command = new Command() "--parent-workspace-id ", "Parent workspace id, used to root a fork-of-a-fork branch", ) + .option( + "--dev-workspace-label ", + "Environment label of a dev workspace (dev/staging); its deploys go to that branch", + ) + .option( + "--parent-dev-workspace-label ", + "Environment label of the parent dev workspace; roots a fork-of-dev branch on it", + ) .option("--skip-secrets", "Skip syncing only secrets variables") .option( "--git-committer-email ", diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index 1cadc8d9d2..83b62f9eb4 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -9,6 +9,7 @@ import { HttpTrigger, KafkaTrigger, MqttTrigger, + AmqpTrigger, NatsTrigger, PostgresTrigger, SqsTrigger, @@ -48,6 +49,7 @@ type Trigger = { nats: NatsTrigger; postgres: PostgresTrigger; mqtt: MqttTrigger; + amqp: AmqpTrigger; sqs: SqsTrigger; gcp: GcpTrigger; azure: AzureTrigger; @@ -84,6 +86,7 @@ async function getTrigger( nats: wmill.getNatsTrigger, postgres: wmill.getPostgresTrigger, mqtt: wmill.getMqttTrigger, + amqp: wmill.getAmqpTrigger, sqs: wmill.getSqsTrigger, gcp: wmill.getGcpTrigger, azure: wmill.getAzureTrigger, @@ -114,6 +117,7 @@ async function updateTrigger( nats: wmill.updateNatsTrigger, postgres: wmill.updatePostgresTrigger, mqtt: wmill.updateMqttTrigger, + amqp: wmill.updateAmqpTrigger, sqs: wmill.updateSqsTrigger, gcp: wmill.updateGcpTrigger, azure: wmill.updateAzureTrigger, @@ -142,6 +146,7 @@ async function createTrigger( nats: wmill.createNatsTrigger, postgres: wmill.createPostgresTrigger, mqtt: wmill.createMqttTrigger, + amqp: wmill.createAmqpTrigger, sqs: wmill.createSqsTrigger, gcp: wmill.createGcpTrigger, azure: wmill.createAzureTrigger, @@ -381,6 +386,13 @@ const triggerTemplates: Record> = { subscribe_topics: [], enabled: false, }, + amqp: { + script_path: "", + is_flow: false, + amqp_resource_path: "", + queue_name: "", + enabled: false, + }, sqs: { script_path: "", is_flow: false, @@ -536,6 +548,7 @@ async function list(opts: GlobalOptions & { json?: boolean }) { natsTriggers, postgresTriggers, mqttTriggers, + amqpTriggers, sqsTriggers, gcpTriggers, azureTriggers, @@ -547,6 +560,7 @@ async function list(opts: GlobalOptions & { json?: boolean }) { listOrEmpty(() => wmill.listNatsTriggers({ workspace: ws })), listOrEmpty(() => wmill.listPostgresTriggers({ workspace: ws })), listOrEmpty(() => wmill.listMqttTriggers({ workspace: ws })), + listOrEmpty(() => wmill.listAmqpTriggers({ workspace: ws })), listOrEmpty(() => wmill.listSqsTriggers({ workspace: ws })), listOrEmpty(() => wmill.listGcpTriggers({ workspace: ws })), listOrEmpty(() => wmill.listAzureTriggers({ workspace: ws })), @@ -559,6 +573,7 @@ async function list(opts: GlobalOptions & { json?: boolean }) { ...natsTriggers.map((x) => ({ path: x.path, kind: "nats" })), ...postgresTriggers.map((x) => ({ path: x.path, kind: "postgres" })), ...mqttTriggers.map((x) => ({ path: x.path, kind: "mqtt" })), + ...amqpTriggers.map((x) => ({ path: x.path, kind: "amqp" })), ...sqsTriggers.map((x) => ({ path: x.path, kind: "sqs" })), ...gcpTriggers.map((x) => ({ path: x.path, kind: "gcp" })), ...azureTriggers.map((x) => ({ path: x.path, kind: "azure" })), @@ -643,11 +658,11 @@ const command = new Command() .command("get", "get a trigger's details") .arguments("") .option("--json", "Output as JSON (for piping to jq)") - .option("--kind ", "Trigger kind (http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email). Recommended for faster lookup") + .option("--kind ", "Trigger kind (http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email). Recommended for faster lookup") .action(get as any) .command("new", "create a new trigger locally") .arguments("") - .option("--kind ", "Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)") + .option("--kind ", "Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email)") .action(newTrigger as any) .command( "push", @@ -662,7 +677,7 @@ const command = new Command() .arguments(" ") .option( "--kind ", - "Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email)" + "Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email)" ) .action((async (opts: any, triggerPath: string, email: string) => { const workspace = await resolveWorkspace(opts); diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 6cd9394f19..eb25ef24ea 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.759.0"; +export const VERSION = "1.770.0"; diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index e4b618180f..54e52564d0 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -217,9 +217,10 @@ export async function pushWorkspaceSettings( throw new Error(`Failed to get workspace settings: ${err}`); } - // Exclude read-only fields from comparison (slack_team_id and slack_name are set via OAuth only) - const { slack_team_id: _lst, slack_name: _lsn, ...comparableLocal } = localSettings; - const { slack_team_id: _rst, slack_name: _rsn, ...comparableRemote } = settings; + // Exclude fields that are never applied here: slack_team_id/slack_name are OAuth-only, + // and name is not applied on pull (see below), so a name-only diff stays a no-op. + const { slack_team_id: _lst, slack_name: _lsn, name: _ln, ...comparableLocal } = localSettings; + const { slack_team_id: _rst, slack_name: _rsn, name: _rn, ...comparableRemote } = settings; if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; @@ -364,15 +365,10 @@ export async function pushWorkspaceSettings( }); } - if (localSettings.name != settings.name) { - log.debug(`Updating workspace name...`); - await wmill.changeWorkspaceName({ - workspace, - requestBody: { - new_name: localSettings.name, - }, - }); - } + // Workspace display name is intentionally not applied on pull: settings.yaml is shared + // across a repo's branches, so applying it would let one workspace's name overwrite + // another's when both sync the same repo. It stays in the file (written on push), but a + // live workspace is only renamed by its owner. if (localSettings.mute_critical_alerts != settings.mute_critical_alerts) { log.debug(`Updating mute critical alerts...`); diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 1d060a58cb..100bea6e53 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -74,7 +74,7 @@ You are a helpful assistant that can help with Windmill scripts, flows, apps, an ## Script Writing Guide -You MUST use the \`write-script-\` skill to write or modify scripts in the language specified by the user. Use bun by default. +You MUST use the \`write-script-\` skill to write or modify scripts in the language specified by the user. Use bun by default. For TypeScript, always prefer bun (\`write-script-bun\`) over deno unless the script specifically requires the Deno runtime. For Workflow-as-Code scripts, use the \`write-workflow-as-code\` skill. ## Flow Writing Guide @@ -89,7 +89,7 @@ When a new app needs to be created, YOU run \`wmill app new\` yourself with \`-- ## Triggers -You MUST use the \`triggers\` skill to configure HTTP routes, WebSocket, Kafka, NATS, SQS, MQTT, GCP, Azure, Email, or Postgres CDC triggers. +You MUST use the \`triggers\` skill to configure HTTP routes, WebSocket, Kafka, NATS, SQS, MQTT, AMQP, GCP, Azure, Email, or Postgres CDC triggers. ## Schedules @@ -137,23 +137,24 @@ There are two ways local changes reach the workspace. Pick based on how the repo ### Detecting the setup -Before deploying, check whether this repo has a **GitHub Actions (or other CI) workflow that runs \`wmill sync push\` on push**. That workflow is the signal that pushing a branch will deploy: +A \`git push\` deploys when either mechanism is in place: **server-handled git sync** (the workspace auto-pulls the remote, so Windmill deploys what you push) or a **CI workflow that runs \`wmill sync push\` on push**. Check for either: -- Look for \`.github/workflows/*.yml\` (or other CI configs) that invoke \`wmill sync push\`, \`wmill\` deployment commands, or similar. +- Run \`wmill gitsync-settings status\`. If it reports \`deploy_on_push\`, pushing the current branch deploys via backend auto-pull — no CI needed. +- Otherwise look for a \`.github/workflows/*.yml\` (or other CI config) that invokes \`wmill sync push\` or similar \`wmill\` deployment commands. -If such a workflow exists → **use \`git push\`** (Option A). Otherwise → **use \`wmill sync push\`** directly (Option B). +If either is present → **use \`git push\`** (Option A). If \`status\` reports no auto-pull and there is no CI wiring, **ask the user** how this repo deploys (a CI pipeline you didn't recognize → \`git push\`, or manual → \`wmill sync push\`) rather than assuming — then record the answer (below). Only use \`wmill sync push\` directly (Option B) once you've confirmed there is no deploy-on-push path. -**Save the preference so you don't re-detect it every session.** Once you've determined which option this repo uses (or the user tells you), record it in the **project-specific instructions** section of \`AGENTS.md\` (user-owned — never overwritten by \`wmill refresh prompts\`), e.g. a line like \`Deploy mode: git push (CI runs wmill sync push)\` or \`Deploy mode: wmill sync push (no CI wiring)\`. On later sessions, read that line first and skip the scan. Re-detect only if the CI wiring visibly changed. +**Save the preference so you don't re-detect it every session.** Once you've determined which option this repo uses (or the user tells you), record it in the **project-specific instructions** section of \`AGENTS.md\` (user-owned — never overwritten by \`wmill refresh prompts\`), e.g. a line like \`Deploy mode: git push (backend auto-pull)\`, \`Deploy mode: git push (CI runs wmill sync push)\`, or \`Deploy mode: wmill sync push (no git-push deploy)\`. On later sessions, read that line first and skip the scan. Re-detect only if the wiring visibly changed. -### Option A — \`git push\` (CI is wired to sync) +### Option A — \`git push\` (backend auto-pull or CI deploys on push) -The CI workflow will pick up the commit and run \`wmill sync push\` on the backend, which is how deployments are intended to happen in this repo. Don't bypass it. +Pushing the commit deploys it: the backend auto-pulls the remote, or a CI workflow runs \`wmill sync push\`. This is how deployments are intended to happen in this repo. Don't bypass it. 1. \`git add\` + \`git commit\` the local changes. -2. \`git push\` to the branch the CI runs on. -3. The workflow deploys to the workspace. +2. \`git push\` to the tracked branch (the one the backend pulls or CI runs on). +3. The deploy happens on the backend. -Only fall back to Option B if the user explicitly asks to bypass CI for this change (e.g. CI is broken, urgent hotfix), or if the workflow doesn't cover the current branch. +Only fall back to Option B if the user explicitly asks to bypass this for this change (e.g. the pipeline is broken, urgent hotfix), or if the tracked branch doesn't cover the current branch. ### Option B — \`wmill sync push\` (no CI wiring) diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index d081b84599..75084ce204 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -10,10 +10,10 @@ export const SKILLS: SkillMetadata[] = [ { name: "write-script-ansible", description: "MUST use when writing Ansible playbooks.", languageKey: "ansible" }, { name: "write-script-bash", description: "MUST use when writing Bash scripts.", languageKey: "bash" }, { name: "write-script-bigquery", description: "MUST use when writing BigQuery queries.", languageKey: "bigquery" }, - { name: "write-script-bun", description: "MUST use when writing Bun/TypeScript scripts.", languageKey: "bun" }, + { name: "write-script-bun", description: "MUST use when writing TypeScript scripts. Bun is the default and preferred TypeScript runtime — pick it for TypeScript unless the script specifically needs Deno.", languageKey: "bun" }, { name: "write-script-bunnative", description: "MUST use when writing Bun Native scripts. The script must start with //native to run on the native worker.", languageKey: "bunnative" }, { name: "write-script-csharp", description: "MUST use when writing C# scripts.", languageKey: "csharp" }, - { name: "write-script-deno", description: "MUST use when writing Deno/TypeScript scripts.", languageKey: "deno" }, + { name: "write-script-deno", description: "Use ONLY when a TypeScript script specifically requires the Deno runtime (Deno stdlib or deno.land URL imports). For all other TypeScript, use write-script-bun instead.", languageKey: "deno" }, { name: "write-script-duckdb", description: "MUST use when writing DuckDB queries.", languageKey: "duckdb" }, { name: "write-script-go", description: "MUST use when writing Go scripts.", languageKey: "go" }, { name: "write-script-graphql", description: "MUST use when writing GraphQL queries.", languageKey: "graphql" }, @@ -53,7 +53,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -83,7 +83,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -193,7 +193,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -223,7 +223,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -293,7 +293,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -323,7 +323,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -374,7 +374,7 @@ being buffered, bypassing the 10000-row return cap. `, "write-script-bun": `--- name: write-script-bun -description: MUST use when writing Bun/TypeScript scripts. +description: MUST use when writing TypeScript scripts. Bun is the default and preferred TypeScript runtime — pick it for TypeScript unless the script specifically needs Deno. --- ## CLI Commands @@ -386,7 +386,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -416,7 +416,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -424,7 +424,7 @@ Use \`wmill resource-type list --schema\` to discover available resource types. # TypeScript (Bun) -Bun runtime with full npm ecosystem and fastest execution. +Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. ## Structure @@ -1038,15 +1038,43 @@ workflow(fn: (...args: any[]) => Promise): void /** * Suspend the workflow and wait for an external approval. * - * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage - * URLs before calling this function. + * Pass \`key\` to name the step, then \`getApprovalUrls(key)\` yields the URLs that + * resume exactly this approval — route them through your own channel. Without a + * key the steps are named \`approval\`, \`approval_2\`, ... * * @example - * const urls = await step("urls", () => getResumeUrls()); - * await step("notify", () => sendEmail(urls.approvalPage)); - * const { value, approver } = await waitForApproval({ timeout: 3600 }); + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. + * + * Unlike \`getResumeUrls()\`, which signs a random nonce, these address the very + * \`resume_job\` record the step's built-in approval buttons use, so they are + * stable across replays and safe to embed in a custom notification. + * + * \`stepKey\` must match the \`key\` given to \`waitForApproval\`. Keys must be unique + * within a workflow; reusing one throws rather than silently renaming it. The URL + * only resumes while that step is awaiting approval; used at any other moment it is + * rejected rather than banking a row a different approval would consume. Send it + * ahead of time — approvers just cannot act before the workflow reaches the step. + * + * \`resume\` and \`cancel\` are step-bound; \`approvalPage\` is not — it opens the job's + * approval page, which acts on whichever approval is pending when it is used. + * + * @example + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * await waitForApproval({ key: "manager" }); + */ +async getApprovalUrls(stepKey: string = "approval", approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> /** * Process items in parallel with optional concurrency control. @@ -1150,7 +1178,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -1180,7 +1208,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -1802,15 +1830,43 @@ workflow(fn: (...args: any[]) => Promise): void /** * Suspend the workflow and wait for an external approval. * - * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage - * URLs before calling this function. + * Pass \`key\` to name the step, then \`getApprovalUrls(key)\` yields the URLs that + * resume exactly this approval — route them through your own channel. Without a + * key the steps are named \`approval\`, \`approval_2\`, ... * * @example - * const urls = await step("urls", () => getResumeUrls()); - * await step("notify", () => sendEmail(urls.approvalPage)); - * const { value, approver } = await waitForApproval({ timeout: 3600 }); + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. + * + * Unlike \`getResumeUrls()\`, which signs a random nonce, these address the very + * \`resume_job\` record the step's built-in approval buttons use, so they are + * stable across replays and safe to embed in a custom notification. + * + * \`stepKey\` must match the \`key\` given to \`waitForApproval\`. Keys must be unique + * within a workflow; reusing one throws rather than silently renaming it. The URL + * only resumes while that step is awaiting approval; used at any other moment it is + * rejected rather than banking a row a different approval would consume. Send it + * ahead of time — approvers just cannot act before the workflow reaches the step. + * + * \`resume\` and \`cancel\` are step-bound; \`approvalPage\` is not — it opens the job's + * approval page, which acts on whichever approval is pending when it is used. + * + * @example + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * await waitForApproval({ key: "manager" }); + */ +async getApprovalUrls(stepKey: string = "approval", approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> /** * Process items in parallel with optional concurrency control. @@ -1914,7 +1970,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -1944,7 +2000,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -1994,7 +2050,7 @@ public class Script `, "write-script-deno": `--- name: write-script-deno -description: MUST use when writing Deno/TypeScript scripts. +description: Use ONLY when a TypeScript script specifically requires the Deno runtime (Deno stdlib or deno.land URL imports). For all other TypeScript, use write-script-bun instead. --- ## CLI Commands @@ -2006,7 +2062,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2036,7 +2092,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2046,6 +2102,8 @@ Use \`wmill resource-type list --schema\` to discover available resource types. Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. +**Prefer Bun (\`write-script-bun\`) for TypeScript.** Only use Deno when the script specifically requires the Deno runtime — Deno's standard library or \`deno.land\` URL imports that have no npm equivalent. For all other TypeScript, use Bun instead. + ## Structure Export a single **async** function called \`main\`: @@ -2658,15 +2716,43 @@ workflow(fn: (...args: any[]) => Promise): void /** * Suspend the workflow and wait for an external approval. * - * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage - * URLs before calling this function. + * Pass \`key\` to name the step, then \`getApprovalUrls(key)\` yields the URLs that + * resume exactly this approval — route them through your own channel. Without a + * key the steps are named \`approval\`, \`approval_2\`, ... * * @example - * const urls = await step("urls", () => getResumeUrls()); - * await step("notify", () => sendEmail(urls.approvalPage)); - * const { value, approver } = await waitForApproval({ timeout: 3600 }); + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. + * + * Unlike \`getResumeUrls()\`, which signs a random nonce, these address the very + * \`resume_job\` record the step's built-in approval buttons use, so they are + * stable across replays and safe to embed in a custom notification. + * + * \`stepKey\` must match the \`key\` given to \`waitForApproval\`. Keys must be unique + * within a workflow; reusing one throws rather than silently renaming it. The URL + * only resumes while that step is awaiting approval; used at any other moment it is + * rejected rather than banking a row a different approval would consume. Send it + * ahead of time — approvers just cannot act before the workflow reaches the step. + * + * \`resume\` and \`cancel\` are step-bound; \`approvalPage\` is not — it opens the job's + * approval page, which acts on whichever approval is pending when it is used. + * + * @example + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * await waitForApproval({ key: "manager" }); + */ +async getApprovalUrls(stepKey: string = "approval", approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> /** * Process items in parallel with optional concurrency control. @@ -2770,7 +2856,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2800,7 +2886,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -2896,7 +2982,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -2926,7 +3012,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3005,7 +3091,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3035,7 +3121,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3101,7 +3187,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3131,7 +3217,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3190,7 +3276,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3220,7 +3306,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3282,7 +3368,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3312,7 +3398,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3375,7 +3461,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3405,7 +3491,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3483,7 +3569,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3513,7 +3599,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3574,7 +3660,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3604,7 +3690,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -3680,7 +3766,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -3710,7 +3796,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4252,6 +4338,17 @@ def get_shared_state(path: str = 'state.json') -> None # Dictionary with approvalPage, resume, and cancel URLs def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict +# Get the resume URLs bound to one \`\`wait_for_approval\`\` step of this workflow. +# +# Args: +# step_key: Checkpoint key of the approval step, as passed to +# \`\`wait_for_approval(key=...)\`\` +# approver: Optional approver name +# +# Returns: +# Dictionary with approvalPage, resume, and cancel URLs +def get_approval_urls(step_key: str = 'approval', approver: str = None) -> dict + # Sends an interactive approval request via Slack, allowing optional customization of the message, approver, and form fields. # # **[Enterprise Edition Only]** To include form fields in the Slack approval request, use the "Advanced -> Suspend -> Form" functionality. @@ -4528,8 +4625,9 @@ async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. # -# Use \`\`get_resume_urls()\`\` (wrapped in \`\`step()\`\`) to obtain -# resume/cancel/approval URLs before calling this function. +# Pass \`\`key\`\` to name the step, then \`\`get_approval_urls(key)\`\` yields the URLs +# that resume exactly this approval — route them through your own channel. +# Without a key the steps are named \`\`approval\`\`, \`\`approval_2\`\`, ... # # Returns a dict with \`\`value\`\` (form data), \`\`approver\`\`, and \`\`approved\`\`. # @@ -4537,13 +4635,14 @@ async def sleep(seconds: int) # timeout: Approval timeout in seconds (default 1800). # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). +# key: Optional checkpoint key naming this approval step. # # Example:: # -# urls = await step("urls", lambda: get_resume_urls()) -# await step("notify", lambda: send_email(urls["approvalPage"])) -# result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict +# urls = await step("urls", lambda: get_approval_urls("manager")) +# await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) +# result = await wait_for_approval(key="manager", timeout=3600) +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict # Process items in parallel with optional concurrency control. # @@ -4583,7 +4682,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4613,7 +4712,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4719,7 +4818,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4749,7 +4848,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4845,7 +4944,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -4875,7 +4974,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -4975,7 +5074,7 @@ After writing, act on the user's intent instead of just listing commands. Run \` - \`wmill flow preview \` — **default when iterating on a local flow.** Runs the local \`flow.yaml\` against local inline scripts without deploying. Add \`--remote\` to use deployed workspace scripts for PathScript steps instead of local files. Add \`--step \` to run only one module in isolation (see "Single-step vs whole-flow preview" below). - \`wmill flow run \` — runs the flow **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate stale local \`.lock\` files for the flow and its inline scripts and refresh their content hashes in \`wmill-lock.yaml\`. Writes local files only (not a deploy). Run it after editing inline scripts whose imports or arguments changed, so \`wmill-lock.yaml\` doesn't drift and add noise to git-sync/CI. By default it scans **scripts, flows, and apps** across the workspace but only regenerates stale ones; pass the flow's folder as an argument (or run from that subdirectory) to limit the scope to the flow you edited. Note a flow (or script) that imports a changed shared script is pulled in too — run \`wmill generate-metadata --dry-run\` to see exactly what is stale and why (\`content changed\` vs \`depends on \`) before applying. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -5585,7 +5684,21 @@ A raw app has three logical parts: ### Entrypoint -\`index.tsx\` is the bundling entrypoint. It typically renders a top-level \`App\` component. The bundler is esbuild. +The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim. + +React (\`index.tsx\`): + +\`\`\`tsx +import React from 'react' +import { createRoot } from 'react-dom/client' +import App from './App' + +createRoot(document.getElementById('root')!).render() +\`\`\` + +Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. Vue (\`index.ts\`): \`createApp(App).mount('#root')\`. + +**Never replace the entrypoint with a bare component** (\`export default function App() { ... }\` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. **Always begin every React file (\`.tsx\`/\`.jsx\`) that uses JSX with \`import React from 'react'\`.** esbuild uses the classic JSX transform, so \`React\` must be in scope wherever JSX appears — a missing import compiles fine but throws \`React is not defined\` at runtime, leaving a blank screen. @@ -5772,7 +5885,7 @@ Text/HTML/inline parts are placed inline in \`body\` as strings. ## CLI Commands -\`wmill sync push\` deploys local changes to the workspace and can be destructive to remote state — only suggest/run it when the user explicitly asks to deploy/publish/push, not when they say "run", "try", or "test". \`sync pull\` is safe to run yourself — it never mutates remote state, though it does overwrite local files to match the remote (use \`sync pull --dry-run\` to only preview). +Deploying local changes to the workspace can be destructive to remote state — only suggest/run a deploy when the user explicitly asks to deploy/publish/push, not when they say "run", "try", or "test". Deploy via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). \`sync pull\` is safe to run yourself — it never mutates remote state, though it does overwrite local files to match the remote (use \`sync pull --dry-run\` to only preview). \`\`\`bash # Push trigger configuration — only when the user explicitly asks to deploy @@ -5823,7 +5936,7 @@ Windmill uses 6-field cron expressions (includes seconds): ## CLI Commands -\`wmill sync push\` deploys local changes to the workspace and can be destructive to remote state — only suggest/run it when the user explicitly asks to deploy/publish/push, not when they say "run", "try", or "test". The commands below never mutate remote state, so they're safe to run yourself — note that \`sync pull\` does overwrite local files to match the remote (use \`sync pull --dry-run\` to only preview), while \`schedule\` just lists. +Deploying local changes to the workspace can be destructive to remote state — only suggest/run a deploy when the user explicitly asks to deploy/publish/push, not when they say "run", "try", or "test". Deploy via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). The commands below never mutate remote state, so they're safe to run yourself — note that \`sync pull\` does overwrite local files to match the remote (use \`sync pull --dry-run\` to only preview), while \`schedule\` just lists. \`\`\`bash # Push schedules to Windmill — only when the user explicitly asks to deploy @@ -6080,8 +6193,9 @@ wmill resource-type list --schema # Get specific resource type schema wmill resource-type get postgresql -# Push resources to Windmill — deploys to the workspace and can be destructive to -# remote state, so only run it when the user explicitly asks to deploy/publish/push +# Deploy resources to the workspace — destructive to remote state, so only run when +# the user explicitly asks to deploy/publish/push. Depending on how the repo is wired, +# deploy via \`git push\` or \`wmill sync push\` (see the Deploying section in AGENTS.wmill.md). wmill sync push \`\`\` `, @@ -6099,7 +6213,7 @@ After writing, tell the user which command fits what they want to do: - \`wmill script preview \` — **default when iterating on a local script.** Runs the local file without deploying. - \`wmill script run \` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. - \`wmill generate-metadata\` — regenerate the local \`.script.yaml\` (input schema) and \`.lock\` (resolved dependencies) for scripts you changed, and refresh their content hashes in \`wmill-lock.yaml\`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- \`wmill sync push\` — deploy local changes to the workspace. Only suggest/run this when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". +- Deploy local changes to the workspace — via \`git push\` or \`wmill sync push\` depending on how the repo is wired (see the **Deploying** section in \`AGENTS.wmill.md\`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". ### Preview vs run — choose by intent, not habit @@ -6129,7 +6243,7 @@ If the user hasn't already told you to run/test/preview the script, offer it as If the user already asked to test/run/try the script in their original request, skip the offer and just execute \`wmill script preview -d ''\` directly — pick plausible args from the script's declared parameters. The shape varies by language: \`main(...)\` for code languages, the SQL dialect's own placeholder syntax (\`$1\` for PostgreSQL, \`?\` for MySQL/Snowflake, \`@P1\` for MSSQL, \`@name\` for BigQuery, etc.), positional \`$1\`, \`$2\`, … for Bash, \`param(...)\` for PowerShell. -\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Only \`wmill sync push\` deploys to the workspace — run it only when the user explicitly asks to deploy/publish/push. +\`wmill script preview\` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). \`wmill generate-metadata\` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's \`AGENTS.md\` opts in), per "Keep metadata in sync" above. Deploying to the workspace (\`git push\` or \`wmill sync push\` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. For a **visual** open-the-script-in-the-dev-page preview (rather than \`script preview\`'s run-and-print-result), use the \`preview\` skill. @@ -6160,7 +6274,7 @@ import { step, sleep, waitForApproval, - getResumeUrls, + getApprovalUrls, parallel, workflow, } from "windmill-client"; @@ -6178,7 +6292,7 @@ export const main = workflow(async (x: string) => { Python: \`\`\`python -from wmill import task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, workflow +from wmill import task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, parallel, workflow @task() async def process(x: str) -> str: @@ -6262,12 +6376,13 @@ output = await pipeline(input=data) Use \`step()\` for lightweight inline values that must not change during replay: \`\`\`typescript -const urls = await step("get_urls", () => getResumeUrls()); const startedAt = await step("started_at", () => new Date().toISOString()); \`\`\` \`\`\`python -urls = await step("get_urls", lambda: get_resume_urls()) +from datetime import datetime + +started_at = await step("started_at", lambda: datetime.now().isoformat()) \`\`\` Use stable, descriptive step names. Do not generate step names dynamically. @@ -6292,20 +6407,28 @@ Only parallelize independent steps. Do not read the result of a task before it i ## Approvals -Generate resume URLs inside \`step()\` before sending them: +Name the approval step and generate its URLs inside \`step()\` before sending them. +\`getApprovalUrls\` / \`get_approval_urls\` returns the URLs bound to that step, the same +ones its built-in approve/reject buttons use: \`\`\`typescript -const urls = await step("get_urls", () => getResumeUrls()); -await step("notify", () => sendApprovalEmail(urls.approvalPage)); -const approval = await waitForApproval({ timeout: 3600 }); +const urls = await step("urls", () => getApprovalUrls("manager")); +await step("notify", () => sendApprovalEmail(urls.resume, urls.cancel)); +const approval = await waitForApproval({ key: "manager", timeout: 3600 }); \`\`\` \`\`\`python -urls = await step("get_urls", lambda: get_resume_urls()) -await step("notify", lambda: send_approval_email(urls["approvalPage"])) -approval = await wait_for_approval(timeout=3600) +urls = await step("urls", lambda: get_approval_urls("manager")) +await step("notify", lambda: send_approval_email(urls["resume"], urls["cancel"])) +approval = await wait_for_approval(key="manager", timeout=3600) \`\`\` +With several approvals in one workflow, give each its own key so each notification +resumes its own step. Keys must be unique — reusing one raises an error rather than +silently renaming the step. A minted URL only resumes while its own step is awaiting +approval; used at any other moment it is rejected rather than resuming the wrong one. \`getResumeUrls()\` / \`get_resume_urls()\` still works but signs a +random nonce, so its URLs are not tied to any particular approval step. + \`selfApproval: false\` and \`self_approval=False\` are Enterprise-only approval behavior. Do not use them unless the user asks for that behavior. ## Error Handling @@ -6319,7 +6442,7 @@ TypeScript: avoid broad \`try/catch\` around WAC SDK calls. The SDK uses an inte ## TypeScript Workflow-as-Code API (windmill-client) -Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getResumeUrls, parallel } from "windmill-client"\` +Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` \`\`\`typescript export interface TaskOptions { @@ -6389,15 +6512,39 @@ export async function sleep(seconds: number): Promise /** * Suspend the workflow and wait for an external approval. * - * Use \`getResumeUrls()\` (wrapped in \`step()\`) to obtain resume/cancel/approvalPage - * URLs before calling this function. + * Pass \`key\` to name the step, then \`getApprovalUrls(key)\` yields the URLs that + * resume exactly this approval — route them through your own channel. Without a + * key the steps are named \`approval\`, \`approval_2\`, ... * * @example - * const urls = await step("urls", () => getResumeUrls()); - * await step("notify", () => sendEmail(urls.approvalPage)); - * const { value, approver } = await waitForApproval({ timeout: 3600 }); + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * const { value, approver } = await waitForApproval({ key: "manager", timeout: 3600 }); */ -export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +export function waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; key?: string; }): PromiseLike<{ value: any; approver: string; approved: boolean }> + +/** + * Resume/cancel/approval-page URLs bound to one \`waitForApproval\` step. + * + * Unlike \`getResumeUrls()\`, which signs a random nonce, these address the very + * \`resume_job\` record the step's built-in approval buttons use, so they are + * stable across replays and safe to embed in a custom notification. + * + * \`stepKey\` must match the \`key\` given to \`waitForApproval\`. Keys must be unique + * within a workflow; reusing one throws rather than silently renaming it. The URL + * only resumes while that step is awaiting approval; used at any other moment it is + * rejected rather than banking a row a different approval would consume. Send it + * ahead of time — approvers just cannot act before the workflow reaches the step. + * + * \`resume\` and \`cancel\` are step-bound; \`approvalPage\` is not — it opens the job's + * approval page, which acts on whichever approval is pending when it is used. + * + * @example + * const urls = await step("urls", () => getApprovalUrls("manager")); + * await step("notify", () => sendEmail(urls.resume, urls.cancel)); + * await waitForApproval({ key: "manager" }); + */ +export async function getApprovalUrls(stepKey: string = "approval", approver?: string): Promise<{ approvalPage: string; resume: string; cancel: string; }> /** * Process items in parallel with optional concurrency control. @@ -6415,7 +6562,7 @@ export async function parallel(items: T[], fn: (item: T) => PromiseLike ## Python Workflow-as-Code API (wmill) -Import: \`from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_resume_urls, parallel, TaskError\` +Import: \`from wmill import workflow, task, task_script, task_flow, step, sleep, wait_for_approval, get_approval_urls, get_resume_urls, parallel, TaskError\` \`\`\`python # Raised when a WAC task step failed. @@ -6503,8 +6650,9 @@ async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. # -# Use \`\`get_resume_urls()\`\` (wrapped in \`\`step()\`\`) to obtain -# resume/cancel/approval URLs before calling this function. +# Pass \`\`key\`\` to name the step, then \`\`get_approval_urls(key)\`\` yields the URLs +# that resume exactly this approval — route them through your own channel. +# Without a key the steps are named \`\`approval\`\`, \`\`approval_2\`\`, ... # # Returns a dict with \`\`value\`\` (form data), \`\`approver\`\`, and \`\`approved\`\`. # @@ -6512,13 +6660,37 @@ async def sleep(seconds: int) # timeout: Approval timeout in seconds (default 1800). # form: Optional form schema for the approval page. # self_approval: Whether the user who triggered the flow can approve it (default True). +# key: Optional checkpoint key naming this approval step. # # Example:: # -# urls = await step("urls", lambda: get_resume_urls()) -# await step("notify", lambda: send_email(urls["approvalPage"])) -# result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict +# urls = await step("urls", lambda: get_approval_urls("manager")) +# await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) +# result = await wait_for_approval(key="manager", timeout=3600) +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True, key: str | None = None) -> dict + +# Get the resume/cancel/approval-page URLs bound to one \`\`wait_for_approval\`\` step. +# +# Unlike :func:\`get_resume_urls\`, which signs a random nonce, these address the +# very \`\`resume_job\`\` record the step's built-in approval buttons use, so they +# are stable across replays and safe to embed in a custom notification. +# +# Args: +# step_key: Checkpoint key of the approval step, as passed to +# \`\`wait_for_approval(key=...)\`\`. Keys must be unique within a workflow; +# reusing one raises rather than silently renaming it. The URL only +# resumes while that step is awaiting approval; used at any other moment +# it is rejected rather than banking a row a different approval would +# consume. Send it ahead of time — approvers just cannot act before the +# workflow reaches the step. +# \`\`resume\`\` and \`\`cancel\`\` are step-bound; \`\`approvalPage\`\` is not — it +# opens the job's approval page, which acts on whichever approval is +# pending when it is used. +# approver: Optional approver name +# +# Returns: +# Dictionary with approvalPage, resume, and cancel URLs +def get_approval_urls(step_key: str = 'approval', approver: str = None) -> dict # Process items in parallel with optional concurrency control. # @@ -6793,6 +6965,8 @@ Manage git-sync settings between local wmill.yaml and Windmill backend - \`--with-backend-settings \` - Use provided JSON settings instead of querying backend (for testing) - \`--yes\` - Skip interactive prompts and use default behavior - \`--promotion \` - Use promotionOverrides from the specified branch instead of regular overrides +- \`gitsync-settings status\` - Report how local changes deploy to the workspace (git push vs wmill sync push) + - \`--json-output\` - Output in JSON format ### group @@ -7215,12 +7389,12 @@ trigger related commands - \`--json\` - Output as JSON (for piping to jq) - \`trigger get \` - get a trigger's details - \`--json\` - Output as JSON (for piping to jq) - - \`--kind \` - Trigger kind (http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email). Recommended for faster lookup + - \`--kind \` - Trigger kind (http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email). Recommended for faster lookup - \`trigger new \` - create a new trigger locally - - \`--kind \` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email) + - \`--kind \` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email) - \`trigger push \` - push a local trigger spec. This overrides any remote versions. - \`trigger set-permissioned-as \` - Set the email (run-as user) for a trigger (requires admin or wm_deployers group) - - \`--kind \` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, sqs, gcp, azure, email) + - \`--kind \` - Trigger kind (required: http, websocket, kafka, nats, postgres, mqtt, amqp, sqs, gcp, azure, email) ### user @@ -7492,6 +7666,120 @@ Both print the job result, are safe to run yourself, and don't deploy. // YAML schema content for triggers and schedules export const SCHEMAS: Record = { + "amqp_trigger": `type: object +properties: + script_path: + type: string + description: Path to the script or flow to execute when triggered + permissioned_as: + type: string + description: The user or group this trigger runs as (permissioned_as) + is_flow: + type: boolean + description: True if script_path points to a flow, false if it points to a script + labels: + type: array + items: + type: string + draft_only: + type: boolean + description: 'True when this row is a per-user draft with no deployed + + trigger at the same path. Set by list endpoints when + + \`include_draft_only=true\` synthesizes the row from the + + draft. Frontend renders a "Draft" badge. + + ' + is_draft: + type: boolean + description: 'True when the authed user has a per-user draft at this path + + (over a deployed row or a synthesized draft-only row). + + Frontend appends a \`*\` to the displayed name. + + ' + amqp_resource_path: + type: string + description: Path to the AMQP resource containing broker connection configuration + queue_name: + type: string + description: Name of the queue to consume messages from + exchange: + type: object + properties: + exchange_name: + type: string + description: Name of the exchange to bind the consumed queue to + routing_keys: + type: array + items: + type: string + description: Routing keys used to bind the queue to the exchange + options: + type: object + properties: + declare_queue: + type: boolean + description: Declare the queue (durable) before consuming; when false the + queue is declared passively and must already exist + prefetch_count: + type: integer + format: int32 + minimum: 1 + maximum: 65535 + description: Maximum number of unacknowledged messages the broker delivers + at once (1-65535) + error_handler_path: + type: string + description: Path to a script or flow to run when the triggered job fails + error_handler_args: + type: object + description: The arguments to pass to the script or flow + retry: + type: object + properties: + constant: + type: object + description: Retry with constant delay between attempts + properties: + attempts: + type: integer + description: Number of retry attempts + seconds: + type: integer + description: Seconds to wait between retries + exponential: + type: object + description: Retry with exponential backoff (delay doubles each time) + properties: + attempts: + type: integer + description: Number of retry attempts + multiplier: + type: integer + description: Multiplier for exponential backoff + seconds: + type: integer + minimum: 1 + description: Initial delay in seconds + random_factor: + type: integer + minimum: 0 + maximum: 100 + description: Random jitter percentage (0-100) to avoid thundering herd + retry_if: + $ref: '#/components/schemas/RetryIf' + description: Retry configuration for failed module executions +required: +- script_path +- permissioned_as +- is_flow +- amqp_resource_path +- queue_name +`, "azure_trigger": `type: object properties: script_path: @@ -8798,6 +9086,7 @@ export const SCHEMA_MAPPINGS: Record = { { name: "NatsTrigger", schemaKey: "nats_trigger", filePattern: "*.nats_trigger.yaml" }, { name: "PostgresTrigger", schemaKey: "postgres_trigger", filePattern: "*.postgres_trigger.yaml" }, { name: "MqttTrigger", schemaKey: "mqtt_trigger", filePattern: "*.mqtt_trigger.yaml" }, + { name: "AmqpTrigger", schemaKey: "amqp_trigger", filePattern: "*.amqp_trigger.yaml" }, { name: "SqsTrigger", schemaKey: "sqs_trigger", filePattern: "*.sqs_trigger.yaml" }, { name: "GcpTrigger", schemaKey: "gcp_trigger", filePattern: "*.gcp_trigger.yaml" }, { name: "AzureTrigger", schemaKey: "azure_trigger", filePattern: "*.azure_trigger.yaml" }, diff --git a/cli/src/types.ts b/cli/src/types.ts index c125ae2c2e..f0cd58abbb 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -63,6 +63,7 @@ export const TRIGGER_TYPES = [ "nats", "postgres", "mqtt", + "amqp", "sqs", "gcp", "azure", @@ -243,6 +244,8 @@ export async function pushObj( await pushTrigger("postgres", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "mqtt_trigger") { await pushTrigger("mqtt", workspace, p, befObj, newObj, permissionedAsContext); + } else if (typeEnding === "amqp_trigger") { + await pushTrigger("amqp", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "sqs_trigger") { await pushTrigger("sqs", workspace, p, befObj, newObj, permissionedAsContext); } else if (typeEnding === "gcp_trigger") { @@ -335,6 +338,7 @@ export function getTypeStrFromPath( | "nats_trigger" | "postgres_trigger" | "mqtt_trigger" + | "amqp_trigger" | "sqs_trigger" | "gcp_trigger" | "azure_trigger" @@ -419,6 +423,7 @@ export function getTypeStrFromPath( typeEnding === "nats_trigger" || typeEnding === "postgres_trigger" || typeEnding === "mqtt_trigger" || + typeEnding === "amqp_trigger" || typeEnding === "sqs_trigger" || typeEnding === "gcp_trigger" || typeEnding === "azure_trigger" || diff --git a/cli/src/utils/git.ts b/cli/src/utils/git.ts index 65e4064a29..da1ac14414 100644 --- a/cli/src/utils/git.ts +++ b/cli/src/utils/git.ts @@ -107,25 +107,42 @@ export interface GitSyncDeployItem { commit_msg?: string; } -// A workspace id of the form "wm-fork-" is a fork workspace. The hub -// script force-disables use_individual_branch / group_by_folder for forks -// (a fork always syncs to its own wm-fork// branch), and that -// disabling also changes the include/promotion derivation — so callers must -// apply it BEFORE deriving includes, not just for branch naming. -export function isForkWorkspace(workspaceId: string): boolean { - return workspaceId.startsWith(FORK_WORKSPACE_PREFIX); +// A throwaway fork syncs to its own wm-fork// branch. The hub script +// force-disables use_individual_branch / group_by_folder for these (that disabling +// also changes the include/promotion derivation — so callers must apply it BEFORE +// deriving includes, not just for branch naming). A dev workspace is the exception: +// with promotion on it keeps use_individual_branch / group_by_folder and gets +// per-item wm_deploy/** branches like a root workspace. +// +// Fork-ness is "has a parent workspace" OR the "wm-fork-" id prefix. Regular forks +// get an auto-generated `wm-fork-` id, but dev workspaces keep a custom id +// with no prefix — so the prefix alone misses them. Mirrors the backend's +// `parent.is_some() || starts_with(WM_FORK_PREFIX)` rule (the prefix also covers a +// fork whose parent was deleted, since parent_workspace_id is ON DELETE SET NULL). +export function isForkWorkspace( + workspaceId: string, + parentWorkspaceId?: string | null, +): boolean { + return !!parentWorkspaceId || workspaceId.startsWith(FORK_WORKSPACE_PREFIX); } -// Mirrors the hub script's get_fork_branch_name: "wm-fork-" becomes -// "wm-fork//". +// Mirrors the hub script's get_fork_branch_name. A dev workspace syncs with its +// environment-label branch verbatim ("dev"/"staging" — a first-class top-level +// branch; the backend passes the label with the deploy). A `wm-fork-` +// throwaway fork id becomes "wm-fork//"; a prefix-less id +// without a label falls back to "wm-fork//". export function forkBranchName( workspaceId: string, originalBranch: string, + devWorkspaceLabel?: string | null, ): string { - return workspaceId.replace( - FORK_WORKSPACE_PREFIX, - `${WM_FORK_PREFIX}/${originalBranch}/`, - ); + if (devWorkspaceLabel) { + return devWorkspaceLabel; + } + const branchPrefix = `${WM_FORK_PREFIX}/${originalBranch}/`; + return workspaceId.startsWith(FORK_WORKSPACE_PREFIX) + ? workspaceId.replace(FORK_WORKSPACE_PREFIX, branchPrefix) + : `${branchPrefix}${workspaceId}`; } // Pure branch-name resolution mirroring the hub script's git_checkout_branch. @@ -133,6 +150,8 @@ export function forkBranchName( // workspace-wide mode, or user/group objects which never get their own branch). export function computeGitSyncDeployBranch(params: { workspaceId: string; + parentWorkspaceId?: string | null; + devWorkspaceLabel?: string | null; items: GitSyncDeployItem[]; useIndividualBranch: boolean; groupByFolder: boolean; @@ -140,17 +159,36 @@ export function computeGitSyncDeployBranch(params: { }): string | null { const { workspaceId, + parentWorkspaceId, + devWorkspaceLabel, items, useIndividualBranch, groupByFolder, clonedBranchName, } = params; - if (workspaceId.startsWith(FORK_WORKSPACE_PREFIX)) { - return forkBranchName(workspaceId, clonedBranchName); + // A dev workspace in promotion mode falls through to the wm_deploy/** formula + // below (per-item/-folder PRs that promote into its parent). Throwaway forks, + // and dev workspaces with promotion off, sync to their own wm-fork// + // (or env-label) branch. + const isDevWorkspace = !!devWorkspaceLabel; + if ( + isForkWorkspace(workspaceId, parentWorkspaceId) && + !(isDevWorkspace && useIndividualBranch) + ) { + return forkBranchName(workspaceId, clonedBranchName, devWorkspaceLabel); } - if (items.length === 0) return null; + // A dev workspace's deploys must never fall through to the base branch — that + // is its parent's tracked branch, so a null here would push dev content + // straight to prod. Anything without its own wm_deploy/** branch (user/group + // objects, an unresolvable ref) goes to the dev's env-label branch instead. + // A root workspace has no such isolation, so its fallback stays null (base). + const fallback = isDevWorkspace + ? forkBranchName(workspaceId, clonedBranchName, devWorkspaceLabel) + : null; + + if (items.length === 0) return fallback; const first = items[0]; // `use_individual_branch` disables debouncing, so items is length 1 here. @@ -159,11 +197,16 @@ export function computeGitSyncDeployBranch(params: { first.path_type === "user" || first.path_type === "group" ) { - return null; + return fallback; } - const ref = first.path ?? first.parent_path; - if (!ref) return null; + // `||` not `??`: the backend serializes a path that no longer matches the repo + // filter (a rename out of the included set) as "" with the old path in + // parent_path. That "" must fall back to parent_path — mirroring the backend's + // `!item_path.is_empty()` derivation. `??` would keep "", return null, and skip + // the branch checkout, letting the removal land on the tracked base branch. + const ref = first.path || first.parent_path; + if (!ref) return fallback; return groupByFolder ? `wm_deploy/${workspaceId}/${ref.split("/").slice(0, 2).join("__")}` @@ -266,6 +309,8 @@ export function gitSyncIncludePattern( return `${path}.postgres_trigger.*`; case "mqtttrigger": return `${path}.mqtt_trigger.*`; + case "amqptrigger": + return `${path}.amqp_trigger.*`; case "sqstrigger": return `${path}.sqs_trigger.*`; case "gcptrigger": diff --git a/cli/test/git_unit.test.ts b/cli/test/git_unit.test.ts index eb4772c6ee..9665b8581a 100644 --- a/cli/test/git_unit.test.ts +++ b/cli/test/git_unit.test.ts @@ -104,6 +104,63 @@ describe("computeGitSyncDeployBranch", () => { expect(branch).not.toBe("main"); }); + test("dev workspace (parent + label) deploys to the label branch", () => { + expect( + computeGitSyncDeployBranch({ + ...base, + workspaceId: "staging-ws", + parentWorkspaceId: "prod", + devWorkspaceLabel: "staging", + useIndividualBranch: false, + items: [{ path_type: "script", path: "f/foo/bar" }], + }) + ).toBe("staging"); + }); + + test("dev workspace in promotion mode -> per-item wm_deploy branch, not the label branch", () => { + expect( + computeGitSyncDeployBranch({ + ...base, + workspaceId: "staging-ws", + parentWorkspaceId: "prod", + devWorkspaceLabel: "staging", + useIndividualBranch: true, + items: [{ path_type: "script", path: "f/foo/bar" }], + }) + ).toBe("wm_deploy/staging-ws/script/f__foo__bar"); + }); + + test("dev promotion user/group objects go to the env-label branch, never the base", () => { + // Non-branchable objects must not fall through to null (= the parent's + // tracked branch) on a dev workspace — that would push dev content to prod. + for (const path_type of ["user", "group"]) { + expect( + computeGitSyncDeployBranch({ + ...base, + workspaceId: "staging-ws", + parentWorkspaceId: "prod", + devWorkspaceLabel: "staging", + useIndividualBranch: true, + items: [{ path_type, path: "u/alice", parent_path: null }], + }) + ).toBe("staging"); + } + }); + + test("dev workspace in promotion mode honors group_by_folder", () => { + expect( + computeGitSyncDeployBranch({ + ...base, + workspaceId: "staging-ws", + parentWorkspaceId: "prod", + devWorkspaceLabel: "staging", + useIndividualBranch: true, + groupByFolder: true, + items: [{ path_type: "script", path: "f/foo/bar" }], + }) + ).toBe("wm_deploy/staging-ws/f__foo"); + }); + test("use_individual_branch=false -> null (stay on base/main, workspace-wide mode)", () => { expect( computeGitSyncDeployBranch({ @@ -135,6 +192,21 @@ describe("computeGitSyncDeployBranch", () => { ).toBe("wm_deploy/prod/flow/f__x__y"); }); + test("falls back to parent_path when the backend serializes path as \"\" (rename out of filter)", () => { + // The backend emits "" (not null) for a path that no longer matches the repo + // filter; it must still get its own branch, not fall through to the base. + expect( + computeGitSyncDeployBranch({ + ...base, + workspaceId: "staging-ws", + parentWorkspaceId: "prod", + devWorkspaceLabel: "staging", + useIndividualBranch: true, + items: [{ path_type: "resource", path: "", parent_path: "f/folder/old" }], + }) + ).toBe("wm_deploy/staging-ws/resource/f__folder__old"); + }); + test("user/group objects never get a dedicated branch", () => { expect( computeGitSyncDeployBranch({ @@ -173,12 +245,54 @@ describe("computeGitSyncDeployBranch", () => { }) ).toBe("wm-fork/main/myfork"); }); + + test("prefix-less fork (parent set, no label) beats the wm_deploy derivation", () => { + expect( + computeGitSyncDeployBranch({ + workspaceId: "mydev", + parentWorkspaceId: "prod", + clonedBranchName: "main", + groupByFolder: false, + useIndividualBranch: true, + items: [{ path_type: "script", path: "f/foo/bar" }], + }) + ).toBe("wm-fork/main/mydev"); + }); + + // A throwaway fork OF a dev workspace is named after the tracked (cloned) + // branch, NOT the parent dev's label: the child is not itself a dev + // workspace, so it carries no devWorkspaceLabel. The dev label reaches this + // deploy only as the checkout base + PR target (handled in sync.ts), never as + // the branch name — otherwise the branch would be `wm-fork/
` with `main` as the default, or +a consumer could never resolve the table it reads. + +### Authorization + +`data_metric` has no RLS. Reads are filtered by an `EXISTS` against `script` on the +authed connection, so `script`'s existing folder, group and user policies decide +what a caller sees. The producing script path is therefore part of the key: a +DuckLake path has no folder to authorize against. + +## Consumers + +**Script editor drawer.** A Metrics trigger sits in the editor bar of every DuckDB +script, and in the compact Helpers menu below the width threshold. It lists the +tables that declare metrics so they can be browsed, composes a plain `SELECT` +client-side from a measure/dimension selection, and offers three things to do with +it: copy it (a complete query attaching the lake under `dl`), run it in an embedded +REPL against the lake, or append it to the script (reusing whatever alias the +script already attaches, since a repeated `ATTACH` would not run). The output is +ordinary editable SQL with no link back to the catalog. + +**Agent tool.** The same endpoint is exposed with `x-mcp-tool`, so an agent can ask +what a table or a folder declares and use the declared `expr`/`filter` instead of +inventing an aggregate. + +## Limitations and follow-ups + +- Declarations only exist for materialized DuckLake tables. There is no way to + declare a measure over a Postgres table or an API result. +- The drawer's REPL runs a preview job per execution, so it is a place to check a + metric rather than a dashboard. There is no charting or filter builder. +- The drawer composes SQL in TypeScript while an agent composes its own. They agree + semantically because both read the same declaration, but not byte for byte. That + only becomes a problem if identical SQL is ever required, which is the compiler + above. +- Deploy hard-rejects three things, all because a reader executes the stored text: + an unsafe lake/table path and a measure/dimension body that is not a single SQL + expression (both are stored SQL injection), and a filtered measure whose body is + not a single aggregate call (`sum(a)/count(b) where …` would apply `FILTER` to + only part of the expression and silently produce the wrong number). Everything + else is advisory: + missing-column and non-aggregate-measure warnings come from the separate + `check_schema_contracts` endpoint, which the editor calls fire-and-forget on save. + A CLI or direct-API deploy skips those warnings, and a declaration whose SQL is + otherwise wrong is only discovered when someone runs it. diff --git a/frontend/e2e/pipeline.spec.ts b/frontend/e2e/pipeline.spec.ts new file mode 100644 index 0000000000..fd472dd206 --- /dev/null +++ b/frontend/e2e/pipeline.spec.ts @@ -0,0 +1,103 @@ +import { test, expect, Page } from '@playwright/test' + +// The pipeline surface an AI session builds into: session pipeline tools emit +// annotated scripts (`-- pipeline`, `-- on `, `-- materialize `), +// and the /pipeline/ editor derives the lineage DAG from those +// annotations alone. This test seeds the scripts an AI-built DuckLake pipeline +// would produce and asserts the editor renders every derived node, asset, and +// edge (including the missing-schedule-trigger edge): a deterministic check of +// the graph the AI session relies on, without a live model in the loop. + +const WORKSPACE = 'admins' + +declare const process: any + +function uniqueSuffix(project: string): string { + // Per-project suffix so the three browser projects don't collide on the + // shared dev instance when Playwright runs them in parallel. + return `${process.env.TEST_UNIQUE_ID ?? 'local'}_${project}` +} + +async function seedScript(page: Page, path: string, content: string, summary: string) { + const res = await page.request.post(`/api/w/${WORKSPACE}/scripts/create`, { + data: { path, summary, description: '', content, language: 'duckdb', schema: {} } + }) + expect(res.ok(), `seed ${path}: ${res.status()} ${await res.text()}`).toBeTruthy() +} + +test.describe('Pipeline editor', () => { + // Track what the test seeds so afterAll can remove it (keeps the shared dev/CI + // instance from accumulating a folder + scripts per run). + let seeded: { folder: string; scripts: string[] } | undefined + + test.afterAll(async ({ request }) => { + if (!seeded) return + for (const path of seeded.scripts) { + await request.post(`/api/w/${WORKSPACE}/scripts/delete/p/${path}`).catch(() => {}) + } + await request.delete(`/api/w/${WORKSPACE}/folders/delete/${seeded.folder}`).catch(() => {}) + }) + + test('derives the DAG from annotated pipeline scripts', async ({ page }, testInfo) => { + const suffix = uniqueSuffix(testInfo.project.name) + const folder = `pipeline_e2e_${suffix}` + const ingest = `f/${folder}/orders_ingest` + const daily = `f/${folder}/orders_daily` + const ordersTbl = `main/orders_${suffix}` + const dailyTbl = `main/orders_daily_${suffix}` + seeded = { folder, scripts: [ingest, daily] } + + // Folder may already exist from a prior run; only fail on the seeds. + await page.request.post(`/api/w/${WORKSPACE}/folders/create`, { data: { name: folder } }) + + await seedScript( + page, + ingest, + [ + '-- pipeline', + '-- on schedule', + `-- materialize ducklake://${ordersTbl}`, + "SELECT * FROM read_csv('s3://raw/orders/*.csv')" + ].join('\n'), + 'Ingest orders' + ) + await seedScript( + page, + daily, + [ + '-- pipeline', + `-- on ducklake://${ordersTbl}`, + `-- materialize ducklake://${dailyTbl}`, + `SELECT date_trunc('day', ts) AS day, count(*) AS n FROM ducklake.${ordersTbl.replace('/', '.')} GROUP BY 1` + ].join('\n'), + 'Daily rollup' + ) + + await page.goto(`/pipeline/${folder}`) + + await expect(page.getByRole('heading', { name: 'Pipeline', level: 1 })).toBeVisible() + await expect(page.getByText('2 scripts', { exact: false })).toBeVisible() + + // A long path truncates in the node label, so match the leaf name; the full + // paths are asserted on the edge labels below. + await expect(page.getByText('orders_ingest').first()).toBeVisible() + await expect(page.getByText('orders_daily').first()).toBeVisible() + + // Two edges resolve asynchronously after the initial graph fetch (the s3 read + // is detected from the SQL body at deploy time; the missing-schedule edge is + // synthesized client-side by the page's per-script annotation sweep), so a + // cold-CI failure here points at that async timing, not a missing edge. + const edges = [ + `Edge from asset:s3object:raw/orders/*.csv to script:${ingest}`, + `Edge from script:${ingest} to asset:ducklake:${ordersTbl}`, + `Edge from asset:ducklake:${ordersTbl} to script:${daily}`, + `Edge from script:${daily} to asset:ducklake:${dailyTbl}`, + `Edge from trigger:schedule:missing:${ingest} to script:${ingest}` + ] + for (const name of edges) { + // Edges are SVG groups (no visible box of their own), so assert they + // are rendered into the graph rather than in-viewport visible. + await expect(page.getByRole('group', { name, exact: true }).first()).toBeAttached() + } + }) +}) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8e4d5c8cbd..5f0ff36020 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.770.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.770.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -41,6 +41,7 @@ "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", "diff": "^7.0.0", + "dompurify": "^3.3.1", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", @@ -54,6 +55,7 @@ "mdast-util-find-and-replace": "^3.0.2", "mermaid": "^11.15.0", "minimatch": "^10.0.1", + "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", "monaco-vim": "^0.4.1", @@ -79,15 +81,15 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.749.0", + "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.693.1", "windmill-parser-wasm-r": "1.668.1", - "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", @@ -9390,6 +9392,12 @@ "dev": true, "license": "MIT" }, + "node_modules/modern-screenshot": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/modern-screenshot/-/modern-screenshot-4.7.0.tgz", + "integrity": "sha512-9YxN+ddPSMMlhylOv25VHzXrl9u67QRxoh7+SEewGtgUw7t6hHTrjptSDJUSne9oG4Xk/h2cwG15nIt4Hc9ujg==", + "license": "MIT" + }, "node_modules/monaco-editor": { "name": "@codingame/monaco-vscode-editor-api", "version": "25.0.0", @@ -14321,9 +14329,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.749.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.749.0.tgz", - "integrity": "sha512-gj8g9sWQ0tXKfXso7xJxR56sS8Loe/RsnFy+0af5R8siZeCag9ikquGbQ8d8kOqIK2U8eCNA0LqsU/xAFDJIOg==" + "version": "1.753.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.753.0.tgz", + "integrity": "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", @@ -14331,9 +14339,9 @@ "integrity": "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ==" }, "node_modules/windmill-parser-wasm-go": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.510.1.tgz", - "integrity": "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ==" + "version": "1.761.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.761.0.tgz", + "integrity": "sha512-jNM0kh0U5uSetwBVSmueFs91GTdbn/tffny2XB1LrrSUElP5i+PK7i12zrWNT2q9JUK2L215CbmyfNmkacbo1Q==" }, "node_modules/windmill-parser-wasm-java": { "version": "1.510.1", @@ -14361,9 +14369,9 @@ "integrity": "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.692.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.692.0.tgz", - "integrity": "sha512-BHGTxrinZJ9ef6hFxbKiBqBEr5uqgG/QySOgMA5r1LswO9n/8fyGswr8JcPT2kGaoeoweV6/RQ+RHVaOhosnKw==" + "version": "1.764.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.764.0.tgz", + "integrity": "sha512-V2eFdKD90gqWikOvjl2fwMpFqiFt/21+4iQMbiNJYl7Lm2UiEcEZ4r9bpgJLG4TLOLqvD6+u4Ju3WaytxN2O2w==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/frontend/package.json b/frontend/package.json index 0015541e04..37b3aff639 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.759.0", + "version": "1.770.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -116,6 +116,7 @@ "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", "diff": "^7.0.0", + "dompurify": "^3.3.1", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", @@ -129,6 +130,7 @@ "mdast-util-find-and-replace": "^3.0.2", "mermaid": "^11.15.0", "minimatch": "^10.0.1", + "modern-screenshot": "^4.7.0", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@=25.0.0", "monaco-languageclient": "10.6.0", "monaco-vim": "^0.4.1", @@ -154,15 +156,15 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.749.0", + "windmill-parser-wasm-asset": "1.753.0", "windmill-parser-wasm-csharp": "1.510.1", - "windmill-parser-wasm-go": "1.510.1", + "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", "windmill-parser-wasm-py": "1.693.1", "windmill-parser-wasm-r": "1.668.1", - "windmill-parser-wasm-regex": "1.692.0", + "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", @@ -302,6 +304,11 @@ "svelte": "./package/components/recording/ScriptRecordingReplay.svelte", "default": "./package/components/recording/ScriptRecordingReplay.svelte" }, + "./components/PipelineRecordingReplay.svelte": { + "types": "./package/components/recording/PipelineRecordingReplay.svelte.d.ts", + "svelte": "./package/components/recording/PipelineRecordingReplay.svelte", + "default": "./package/components/recording/PipelineRecordingReplay.svelte" + }, "./components/recording/types": { "types": "./package/components/recording/types.d.ts", "default": "./package/components/recording/types.js" @@ -530,6 +537,9 @@ "components/ScriptRecordingReplay.svelte": [ "./package/components/recording/ScriptRecordingReplay.svelte.d.ts" ], + "components/PipelineRecordingReplay.svelte": [ + "./package/components/recording/PipelineRecordingReplay.svelte.d.ts" + ], "components/recording/types": [ "./package/components/recording/types.d.ts" ], diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index b901dc9bc0..4360794dbe 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "f8cecf9", - "sha256": "2a79b838e21e06abe06119263872afbe83c4a13f7aa72c722af3e34b819f0ce3" + "version": "1f1fe4f", + "sha256": "4c20b2b51f324e93dda3b46d914ebf7bc7d0cb4dc7eb2d37068408191220d206" } diff --git a/frontend/src/lib/appDiffSides.test.ts b/frontend/src/lib/appDiffSides.test.ts new file mode 100644 index 0000000000..72abbbffb9 --- /dev/null +++ b/frontend/src/lib/appDiffSides.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest' +import { classicAppDraftParts } from './appDiffSides' + +describe('classicAppDraftParts', () => { + it('extracts the mirrored summary and staged rename, stripping them from the grid', () => { + const draft = { + grid: [{ id: 'a' }], + fullscreen: false, + summary: 'My app', + draft_path: 'f/nice/name', + parent_version: 7 + } + expect(classicAppDraftParts(draft)).toEqual({ + value: { grid: [{ id: 'a' }], fullscreen: false }, + summary: 'My app', + draftPath: 'f/nice/name' + }) + }) + + it('unwraps a legacy wrapped draft so it compares against the deployed value', () => { + const wrapped = { + summary: 'My app', + policy: {}, + value: { grid: [{ id: 'a' }] } + } + expect(classicAppDraftParts(wrapped)).toEqual({ + value: { grid: [{ id: 'a' }] }, + summary: 'My app' + }) + }) + + it('does not mistake a grid whose component is named value for a wrapper', () => { + // Production shape: the editor ALWAYS mirrors summary onto the bare app. + const grid = { grid: [{ id: 'a' }], value: { some: 'component-state' }, summary: 'My app' } + expect(classicAppDraftParts(grid)).toEqual({ + value: { grid: [{ id: 'a' }], value: { some: 'component-state' } }, + summary: 'My app', + draftPath: undefined + }) + }) +}) diff --git a/frontend/src/lib/appDiffSides.ts b/frontend/src/lib/appDiffSides.ts new file mode 100644 index 0000000000..68ff75c11b --- /dev/null +++ b/frontend/src/lib/appDiffSides.ts @@ -0,0 +1,36 @@ +/** Split a CLASSIC app draft into comparable parts. The editor mirrors + * summary/draft_path INTO the bare grid it autosaves, while a deployed row + * keeps them outside `value` — compared raw, metadata diffs as grid noise. + * Also unwraps legacy wrapped drafts ({summary/policy/custom_path, value}). */ +export function classicAppDraftParts(json: unknown): { + value: unknown + summary?: string + draftPath?: string +} { + if (json === null || typeof json !== 'object' || Array.isArray(json)) { + return { value: json } + } + const obj = json as Record + // A bare App always carries `grid` at top level (and the editor mirrors + // summary into it), while a legacy wrapper never does — `grid` is the only + // reliable discriminator; metadata keys appear on both shapes. + const wrapped = + !('grid' in obj) && + typeof obj.value === 'object' && + obj.value !== null && + 'grid' in (obj.value as Record) + if (wrapped) { + const inner = classicAppDraftParts(obj.value) + return { + value: inner.value, + summary: (obj.summary as string | undefined) ?? inner.summary, + draftPath: (obj.draft_path as string | undefined) ?? inner.draftPath + } + } + const { parent_version: _pv, draft_path, summary, ...value } = obj + return { + value, + summary: summary as string | undefined, + draftPath: draft_path as string | undefined + } +} diff --git a/frontend/src/lib/cloud.ts b/frontend/src/lib/cloud.ts index a684153e7a..350b96aacc 100644 --- a/frontend/src/lib/cloud.ts +++ b/frontend/src/lib/cloud.ts @@ -2,4 +2,18 @@ import { BROWSER } from 'esm-env' export function isCloudHosted(): boolean { return BROWSER && window.location.hostname == 'app.windmill.dev' -} \ No newline at end of file +} + +// On the managed cloud, the public demo workspace is kept clean and consistent by +// disabling folder creation, item sharing, and group creation for non-admins. The +// backend enforces the same rule; this only drives the UI hints. +export const DEMO_RESTRICTION_HINT = + 'Disabled in the demo workspace. Create your own workspace to keep the demo clean and consistent.' + +export function isDemoWorkspaceRestricted( + workspace: string | undefined, + isAdmin: boolean | undefined, + isSuperAdmin: boolean | undefined +): boolean { + return isCloudHosted() && workspace === 'demo' && !isAdmin && !isSuperAdmin +} diff --git a/frontend/src/lib/coalescingRunner.svelte.ts b/frontend/src/lib/coalescingRunner.svelte.ts index 0f687d4829..61849c3e36 100644 --- a/frontend/src/lib/coalescingRunner.svelte.ts +++ b/frontend/src/lib/coalescingRunner.svelte.ts @@ -30,6 +30,9 @@ export type CoalescingKeyedRunner = { cancel(key: string): boolean /** Reactively whether `key`'s chain is running (SvelteSet-backed). */ isRunning(key: string): boolean + /** Resolves once `key`'s chain has drained (nothing running, nothing + * pending), immediately if it's idle. Never rejects. */ + settled(key: string): Promise } type PendingTask = { @@ -51,6 +54,8 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { // Reactive mirror of keys with a running chain, kept in lock-step with // `state` (SvelteSet for per-key `isRunning` subscriptions). const runningKeys = new SvelteSet() + // Live chain promise per key, backing `settled`. + const chains = new Map>() async function chain(key: string, first: PendingTask): Promise { let current: PendingTask | undefined = first @@ -71,6 +76,7 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { } state.delete(key) runningKeys.delete(key) + chains.delete(key) } /** Set `task` pending for `key`, displacing (and rejecting) any prior @@ -84,7 +90,15 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { } state.set(key, { pending: undefined }) runningKeys.add(key) - void chain(key, task) + // Register the chain promise BEFORE the first task runs. `chain` invokes + // the task synchronously, so a task that calls `settled(key)` (or that + // throws synchronously, running cleanup) would otherwise race ahead of a + // `chains.set(key, chain(...))` and leave the map wrong. A separate + // deferred sidesteps that: it's live before the task starts and resolves + // when the chain drains. + let done!: () => void + chains.set(key, new Promise((resolve) => (done = resolve))) + void chain(key, task).finally(done) } function submit(key: string, fn: CoalescingTask): void { @@ -114,5 +128,9 @@ export function createCoalescingKeyedRunner(): CoalescingKeyedRunner { return runningKeys.has(key) } - return { submit, submitAndWait, cancel, isRunning } + function settled(key: string): Promise { + return chains.get(key) ?? Promise.resolve() + } + + return { submit, submitAndWait, cancel, isRunning, settled } } diff --git a/frontend/src/lib/coalescingRunner.test.ts b/frontend/src/lib/coalescingRunner.test.ts index 49cca4688e..1ae97e7db1 100644 --- a/frontend/src/lib/coalescingRunner.test.ts +++ b/frontend/src/lib/coalescingRunner.test.ts @@ -115,6 +115,87 @@ describe('createCoalescingKeyedRunner', () => { expect(runner.cancel('k')).toBe(false) }) + it('settled resolves immediately for an idle key', async () => { + const runner = createCoalescingKeyedRunner() + await expect(runner.settled('k')).resolves.toBeUndefined() + }) + + it('settled resolves once the chain drains, including the displacing task', async () => { + const runner = createCoalescingKeyedRunner() + const d = deferred() + const last = deferred() + const h = vi.fn(() => last.promise) + + runner.submit('k', () => d.promise) // in flight + void runner.submitAndWait('k', () => Promise.resolve()).catch(() => {}) // displaced below + runner.submit('k', h) + + let drained = false + void runner.settled('k').then(() => (drained = true)) + + d.resolve() + await d.promise + await Promise.resolve() + await Promise.resolve() + expect(h).toHaveBeenCalledTimes(1) + expect(drained).toBe(false) // h still running + + last.resolve() + await runner.settled('k') + expect(drained).toBe(true) + expect(runner.isRunning('k')).toBe(false) + }) + + it('settled called synchronously from within the first task does not resolve early', async () => { + const runner = createCoalescingKeyedRunner() + const d = deferred() + let settledEarly = false + let settledResolved = false + runner.submit('k', () => { + // Re-entrant: the task is invoked synchronously as the chain starts. + const p = runner.settled('k') + void p.then(() => (settledResolved = true)) + // Give the microtask a tick to (wrongly) resolve if the entry is missing. + void Promise.resolve().then(() => { + if (settledResolved) settledEarly = true + }) + return d.promise + }) + await Promise.resolve() + await Promise.resolve() + expect(settledEarly).toBe(false) + expect(settledResolved).toBe(false) // still running + + d.resolve() + await runner.settled('k') + expect(settledResolved).toBe(true) + }) + + it('a synchronously-throwing first task leaves no stale chain entry', async () => { + const runner = createCoalescingKeyedRunner() + const err = vi.spyOn(console, 'error').mockImplementation(() => {}) + runner.submit('k', () => { + throw new Error('sync boom') + }) + // Chain drained synchronously; the key must be idle and settled a no-op. + expect(runner.isRunning('k')).toBe(false) + await expect(runner.settled('k')).resolves.toBeUndefined() + // A fresh submit still starts a new chain (map wasn't left stale). + const ran = vi.fn(() => Promise.resolve()) + runner.submit('k', ran) + expect(ran).toHaveBeenCalledTimes(1) + err.mockRestore() + }) + + it('settled ignores a task failure (the chain survives it)', async () => { + const runner = createCoalescingKeyedRunner() + const err = vi.spyOn(console, 'error').mockImplementation(() => {}) + runner.submit('k', () => Promise.reject(new Error('boom'))) + await expect(runner.settled('k')).resolves.toBeUndefined() + expect(runner.isRunning('k')).toBe(false) + err.mockRestore() + }) + it('does not abort the in-flight task on cancel', async () => { const runner = createCoalescingKeyedRunner() const d = deferred() diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index dd02a3c578..c0a25daeb7 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -231,7 +231,7 @@ >No corresponding resource type found in your workspace for {resourceType}. Define the value in JSON directly

- + {/if} {#if notFound || viewJsonSchema} {#if !emptyString(error)} Resource type '{resourceType}' not found in your workspace

- + {/if} {#if registryCcCapable()} diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index cfca54bc99..cda34487af 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -37,9 +37,14 @@ const dispatch = createEventDispatcher() - const customTagRegex = /^([\w-]+)\(((?:[\w-]+\+)*[\w-]+|(?:\^[\w-]+)+)\)$/ + // Mirrors CUSTOM_TAG_REGEX in backend/windmill-common/src/worker.rs — keep both in sync. + const customTagRegex = /^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ const dynamicTagRegex = /\$args\[((?:\w+\.)*\w+)\]/ + function formatWorkspace(w: { id: string; includeForks: boolean }) { + return w.includeForks ? `${w.id} (and its forks)` : w.id + } + let dynamicTag = $derived.by(() => { let r = newTag.trim() if (r == '') return undefined @@ -51,14 +56,16 @@ let r = newTag.trim() if (r == '') return undefined let matched = r.match(customTagRegex) - console.log(matched) let tag = matched?.[1] let workspaces_raw = matched?.[2] let tag_type = workspaces_raw?.includes('^') ? 'exclude' : 'include' if (tag_type == 'exclude') { workspaces_raw = workspaces_raw?.slice(1) } - let workspaces = workspaces_raw?.split(tag_type == 'include' ? '+' : '^') + let workspaces = workspaces_raw?.split(tag_type == 'include' ? '+' : '^').map((w) => { + const includeForks = w.endsWith('*') + return { id: includeForks ? w.slice(0, -1) : w, includeForks } + }) if (!workspaces_raw || workspaces_raw?.length == 0) { return undefined } @@ -163,14 +170,14 @@
Workspaces: {#if extractedCustomTag.tag_type == 'include'} - {extractedCustomTag.workspaces?.join(', ')} + {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {:else} - All workspaces except {extractedCustomTag.workspaces?.join(', ')} + All workspaces except {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {/if}
{:else if newTag.trim()} - {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || ((newTag.includes('.') || newTag.includes('$args[')) && !dynamicTag)} + {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || newTag.includes('*') || ((newTag.includes('.') || newTag.includes('$args[')) && !dynamicTag)}
Invalid tag
@@ -219,6 +226,10 @@ To exclude 'workspace1' and 'workspace2' from a tag, use
tag(^workspace1^workspace2)

{#if variant !== 'drawer'}
{/if} + Forks of a workspace do not get its tags. Suffix a workspace with +
*
+ to also cover its forks, e.g.
tag(workspace1*)
+
{#if variant !== 'drawer'}
{/if} For void onSelectedChannelChange?: (channel: ChannelItem | undefined) => void + /** Workspace to list Teams channels from; defaults to the nav + * `$workspaceStore`. A forked session passes its acting workspace. */ + workspace?: string } let { @@ -33,9 +36,12 @@ teamId, showRefreshButton = true, onError, - onSelectedChannelChange + onSelectedChannelChange, + workspace = undefined }: Props = $props() + let effectiveWorkspace = $derived(workspace ?? $workspaceStore) + let isFetching = $state(false) let loadedChannels = $state([]) let loadedForTeamId = $state(undefined) @@ -88,7 +94,7 @@ isFetching = true try { const response = await WorkspaceService.listAvailableTeamsChannels({ - workspace: $workspaceStore!, + workspace: effectiveWorkspace!, teamId: teamId }) @@ -130,10 +136,7 @@ clearable disabled={disabled || !teamId} loading={isFetching} - bind:value={ - () => selectedChannel?.channel_id, - (newId) => setSelectedChannelById(newId) - } + bind:value={() => selectedChannel?.channel_id, (newId) => setSelectedChannelById(newId)} /> {:else}
@@ -206,7 +242,7 @@ - + { + publishFolderName = name + hubDrawer?.openDrawer() + } + }, { displayName: `Delete${canWrite ? '' : ' (require owner permissions)'}`, icon: Trash, diff --git a/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte b/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte index 6670be1225..152948b190 100644 --- a/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/forks/compare/+page.svelte @@ -2,6 +2,7 @@ import CompareWorkspaces from '$lib/components/CompareWorkspaces.svelte' import CompareDrafts from '$lib/components/CompareDrafts.svelte' import { WorkspaceService, type WorkspaceComparison } from '$lib/gen' + import { fetchWorkspaceComparison } from '$lib/workspaceComparison' import { archiveSessionsForWorkspace, deleteSessionsForWorkspace, @@ -20,6 +21,11 @@ import { switchWorkspace } from '$lib/storeUtils' import { goto } from '$lib/navigation' import { readChatModifiedItems } from '$lib/components/copilot/chat/HistoryManager.svelte' + import { + COMPARE_ITEMS_PARAM, + maskHasDraftRow, + parseItemsMaskParam + } from '$lib/components/sessions/modifiedItemsMask' type CompareMode = 'fork' | 'draft' @@ -49,6 +55,15 @@ // selection here; the page only swaps which comparison component is shown. let forkDirection = $state<'deploy_to' | 'update'>('deploy_to') + // Explicit preselection via `?items=` (built by the chat's + // open_page tool). Parsed synchronously from the live URL so it can never race + // the children's select-all default. Present-but-empty means "preselect + // nothing", distinct from absent (undefined → no mask). + const urlItemsMask = $derived.by(() => { + const v = page.url.searchParams.get(COMPARE_ITEMS_PARAM) + return v === null ? undefined : parseItemsMaskParam(v) + }) + // When reached via a session's Review button (`from_session=`), preselect // only the items that chat modified. The mask is the chat's stored // `${UserDraftItemKind}:${storagePath}` set; undefined for a legacy chat (no @@ -56,30 +71,33 @@ // Derived from the live URL: an in-app navigation to this route with a // different from_session must reload the mask, not keep the first one. const fromChatId = $derived(page.url.searchParams.get('from_session')) - let chatMask = $state | undefined>(undefined) + let sessionMask = $state | undefined>(undefined) // The mask loads asynchronously, while the resolved value can legitimately be // undefined (legacy chat). The children must not run their select-all default // until the mask is known, else they'd race it and select everything. Ready // immediately when there's no session to read from. - let chatMaskReady = $state(!page.url.searchParams.get('from_session')) + let sessionMaskReady = $state(!page.url.searchParams.get('from_session')) $effect(() => { const id = fromChatId - chatMask = undefined - chatMaskReady = !id + sessionMask = undefined + sessionMaskReady = !id if (!id) return untrack(() => { void readChatModifiedItems(id) .then((arr) => { // A slower read for a superseded chat id must not win. if (id !== untrack(() => fromChatId)) return - chatMask = arr ? new Set(arr) : undefined + sessionMask = arr ? new Set(arr) : undefined }) .finally(() => { - if (id === untrack(() => fromChatId)) chatMaskReady = true + if (id === untrack(() => fromChatId)) sessionMaskReady = true }) }) }) + const chatMask = $derived(urlItemsMask ?? sessionMask) + const chatMaskReady = $derived(urlItemsMask !== undefined || sessionMaskReady) + function selectMode(v: 'deploy_to' | 'update' | 'draft') { if (v === 'draft') { mode = 'draft' @@ -92,8 +110,17 @@ // Draft count drives the "Deployed ↔ draft" toggle badge. Reads the shared // Workspace Drafts resource — count ≡ the draft list, and it refreshes itself // when a deploy/discard invalidates the workspace. - const drafts = useWorkspaceDrafts(() => currentWorkspaceId) - const draftCount = $derived(drafts.count) + const drafts = useWorkspaceDrafts( + () => currentWorkspaceId, + () => false, + () => (isFork ? (parentWorkspaceId ?? undefined) : undefined) + ) + // On a fork, match the badge to the default deploy-draft view, which hides + // drafts unchanged from the parent (else a fresh fork shows a count over an + // empty list). + const draftCount = $derived( + isFork ? drafts.items.filter((d) => d.unchanged_from_parent !== true).length : drafts.count + ) // Keys (`kind:path`) of fork items that are deployed *and* carry a pending // draft (has_draft, i.e. not draft_only). CompareWorkspaces uses this to flag @@ -124,8 +151,40 @@ $effect(() => { if (modeResolved || !currentWorkspaceData) return + if (!isFork) { + untrack(() => { + mode = 'draft' + modeResolved = true + }) + return + } + // An explicit ?mode=fork is only deferred (not latched at init) so the + // non-fork fallback above can veto it — on a real fork, honor it as is. + if (urlMode === 'fork') { + untrack(() => { + mode = 'fork' + modeResolved = true + }) + return + } + // A fork reached with a preselection mask but no ?mode= must land on the + // view where the masked items actually are: a chat's pending drafts have no + // fork-diff row, so fork mode would open with none of them selected. Defer + // until the mask and the draft list are known, then prefer the draft view + // when any masked item is a pending draft; else keep the fork comparison. + if (!chatMaskReady) return + const mask = chatMask + if (mask?.size) { + if (drafts.loading) return + const masksDraft = drafts.items.some((d) => maskHasDraftRow(mask, d)) + untrack(() => { + mode = masksDraft ? 'draft' : 'fork' + modeResolved = true + }) + return + } untrack(() => { - mode = isFork ? 'fork' : 'draft' + mode = 'fork' modeResolved = true }) }) @@ -136,10 +195,7 @@ } try { - const result = await WorkspaceService.compareWorkspaces({ - workspace: parentWorkspaceId, - targetWorkspaceId: currentWorkspaceId - }) + const result = await fetchWorkspaceComparison(parentWorkspaceId, currentWorkspaceId) comparison = result } catch (e) { diff --git a/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte b/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte index 82adf2a459..f645f7e3d6 100644 --- a/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/global_drafts/+page.svelte @@ -6,7 +6,6 @@ listGlobalDrafts } from '$lib/components/copilot/chat/global/userDraftAdapter' import type { WorkspaceItem } from '$lib/components/copilot/chat/global/workspaceItems' - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' import { goto } from '$lib/navigation' import { workspaceStore } from '$lib/stores' import { Trash2 } from 'lucide-svelte' @@ -21,8 +20,8 @@ } onMount(() => { - // Dev-only route. Bounce to home when the global mode gate is closed. - enabled = isGlobalAiEnabled() + // Dev tooling, not part of the sessions beta — only reachable on dev builds. + enabled = import.meta.env.DEV if (!enabled) { goto('/') } diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index 0668c4c307..85b4b536fe 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -22,9 +22,14 @@ import { untrack } from 'svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import { Tooltip } from '$lib/components/meltComponents' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' type GroupW = Group & { canWrite: boolean } + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let newGroupName: string = $state('') let groups: GroupW[] | undefined = $state(undefined) let instanceGroups: InstanceGroupWithWorkspaces[] | undefined = $state(undefined) @@ -103,37 +108,49 @@ >
- - {#snippet trigger()} - - {/snippet} - {#snippet content({ close })} -
- handleKeyUp(e, close) - }} - bind:value={newGroupName} - /> - + {:else} + + {#snippet trigger()} + - Create - -
- {/snippet} -
+ {/snippet} + {#snippet content({ close })} +
+ handleKeyUp(e, close) + }} + bind:value={newGroupName} + /> + +
+ {/snippet} + + {/if}
diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 0c1a6eeb8c..6123c07000 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -43,6 +43,7 @@ assetProducers } from '$lib/components/assets/AssetGraph/graphTraversal' import { runCascade, runSelection } from '$lib/components/assets/AssetGraph/cascadeOrchestrator' + import { DATA_ASSET_KINDS } from '$lib/components/assets/AssetGraph/cascadeRun' import { boundedSet, buildLineageDag, @@ -72,6 +73,11 @@ type PipelineDraft } from '$lib/components/assets/AssetGraph/pipelineAiHelpers' import { PipelineEditorState } from '$lib/components/assets/AssetGraph/pipelineEditorState.svelte' + import { + createPipelineRecording, + finalizePipelineRecording + } from '$lib/components/recording/pipelineRecording.svelte' + import type { PipelineRecording } from '$lib/components/recording/types' import AutosaveIndicator from '$lib/components/AutosaveIndicator.svelte' import { onMount, tick, untrack } from 'svelte' import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' @@ -79,6 +85,8 @@ AlertTriangle, ArrowLeft, ChevronDown, + Circle, + Download, Folder, FolderSearch, History, @@ -101,7 +109,7 @@ type ScriptLang } from '$lib/gen' import { resource } from 'runed' - import { emptySchema, sendUserToast } from '$lib/utils' + import { emptySchema, sendUserToast, type Item } from '$lib/utils' import type { Schema } from '$lib/common' import { beforeNavigate, goto } from '$app/navigation' import { fade } from 'svelte/transition' @@ -113,7 +121,7 @@ // Variables and resources are declarative config, not pipeline assets — // they're hub-shaped (referenced by most runnables) and would swamp the // layout without adding lineage information. - const DATA_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] + const DATA_KINDS = DATA_ASSET_KINDS let folder = $derived(page.params.folder as string) @@ -1404,6 +1412,61 @@ // other's storage writes. let cascadeRunningRoot = $state(undefined) + // Recorder: when armed, the next cascade run captures the resolved graph, the + // per-node status timeline and each node's job stream into a downloadable + // recording that the /pipeline_replay player can rerun offline (parity with the + // flow/script recorders). Job capture (`watchJob`) and status capture + // (`recordStatuses`) no-op unless the store is active, so the cascade run + // paths call them unconditionally. + let pipelineRecording = createPipelineRecording() + let recordingMode = $state(false) + let lastPipelineRecording = $state(undefined) + + // Shared by the overflow-menu Record item and the inline armed pill so their + // wording can't drift — both describe the same armed recorder. + const RECORDING_ARMED_HINT = + 'Recording armed — the next pipeline run will be captured. Click to disarm.' + + function downloadPipelineRecording() { + if (lastPipelineRecording) { + pipelineRecording.download(lastPipelineRecording) + } + } + + // Secondary top-bar controls (recorder, macros) collapse into a single + // overflow (⋮) menu so the bar stays legible on small screens; only primary + // actions stay inline. Recording lives here rather than on the bar at all + // times — while armed it surfaces a compact inline pill (below) instead. + let overflowMenuItems = $derived.by(() => { + const items: Item[] = [] + if (!isOperator && allPipelineScripts.length > 0) { + items.push({ + displayName: recordingMode ? 'Disarm recorder' : 'Record next run', + icon: Circle, + iconColor: recordingMode ? 'rgb(220 38 38)' : undefined, + disabled: !!cascadeRunningRoot, + tooltip: recordingMode + ? RECORDING_ARMED_HINT + : 'Arm the recorder so the next pipeline run is captured for offline replay', + action: () => (recordingMode = !recordingMode) + }) + if (lastPipelineRecording && !cascadeRunningRoot) { + items.push({ + displayName: 'Download last recording', + icon: Download, + action: () => downloadPipelineRecording() + }) + } + } + items.push({ + displayName: 'Macros', + icon: SquareFunction, + tooltip: "Browse the workspace's DuckDB macros (deployed // macros libraries)", + action: () => macroDrawer?.openDrawer() + }) + return items + }) + // Script path → its schedule's configured args, so a manual "Run pipeline" // launches a schedule-triggered script with the same payload a real tick // would (rather than empty args). Schedule is the only trigger that stores a @@ -1711,6 +1774,10 @@ // Claim the running-guard BEFORE the first await so a rapid second click // (which reads `cascadeRunningRoot`) can't slip through and double-launch. cascadeRunningRoot = schedule.roots[0] ?? scripts[0] + if (recordingMode) { + lastPipelineRecording = undefined + pipelineRecording.start(folder, displayGraph) + } let firstJobId: string | undefined try { // Seed schedule-triggered roots with their configured payload. @@ -1720,6 +1787,8 @@ launch: async (path) => { const jobId = await launchCascadeScript(path) activeRunnables.arm(`script:${path}`) + // No-op unless a recording is active; captures the node's stream. + if ($workspaceStore) pipelineRecording.watchJob(jobId, $workspaceStore) if (firstJobId === undefined) { firstJobId = jobId runsPendingJobId = jobId @@ -1727,7 +1796,8 @@ } return jobId }, - waitTerminal: waitJobTerminal + waitTerminal: waitJobTerminal, + onUpdate: (statuses) => pipelineRecording.recordStatuses(statuses) }) const n = res.statuses.size if (res.ok) { @@ -1750,7 +1820,21 @@ ) } } finally { - cascadeRunningRoot = undefined + // Hold the run guard until finalization finishes: finalize keeps writing + // jobs/samples/code through the recorder store, and a second run's + // `start()` would reset those maps mid-write, corrupting both recordings. + // The nested finally still clears the guard if finalize ever rejects, so + // Run can't wedge permanently. + try { + if (pipelineRecording.active) { + lastPipelineRecording = await finalizePipelineRecording( + pipelineRecording, + $workspaceStore + ) + } + } finally { + cascadeRunningRoot = undefined + } } } @@ -2326,6 +2410,20 @@ {/if}
{#if !isOperator && allPipelineScripts.length > 0} + + {#if recordingMode} + + {/if} +{#snippet replayFailed()} +
+ +

+ This recording could not be replayed — it may be malformed or from an incompatible version. +

+ +
+{/snippet} + + +
+ {#if flowRecording} +
+ +
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + + {:else if scriptRecording} +
+ +
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + + {:else if pipelineRecording} +
+ +
+
+ setActiveReplay(undefined)}> + + {#snippet failed()}{@render replayFailed()}{/snippet} + +
+ {:else if downloading} +
+
+ +

Downloading recording…

+ {#if downloadPercent !== undefined} +
+
+
+

{downloadPercent}% · {fmtBytes(downloadedBytes)}

+ {:else} +

{fmtBytes(downloadedBytes)}

+ {/if} +
+
+ {:else} +
+
+

Replay a recording

+

+ Upload a recording JSON file to replay a flow, script or data-pipeline execution offline. +

+ {#if downloadError} +

{downloadError}

+ {/if} + + Drag and drop a recording file + +
+
+ {/if} +
diff --git a/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte new file mode 100644 index 0000000000..06f2344356 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/projects/install/+page.svelte @@ -0,0 +1,374 @@ + + +
+ {#if !slug} +

Missing ?hub=<slug>.

+ {:else if loading} +
+ Loading project… +
+ {:else if loadError} +

Failed to load project: {loadError}

+ {:else if data} +

Add “{data.project.name}” to workspace

+

{data.project.summary}

+ +
+

+ Folder in {workspace} +

+ +

+ Items import under f/{folderName.trim() || data.project.slug}/. +

+
+ +
+ {counts?.scripts} scripts + {counts?.flows} flows + {counts?.apps} apps + {counts?.resources} resources + {counts?.triggers} triggers + {#if counts && counts.migrations > 0} + {counts.migrations} data table migrations + {/if} +
+ +
+ Resources are imported as empty stubs — set their values after import; a resource whose path + already exists is reported as failed (existing values are never overwritten). Trigger kinds + are recreated disabled, except GCP and Azure triggers, which manage cloud subscriptions at + creation and must be re-created manually after filling their resource. Kafka, NATS, SQS, GCP + and Azure triggers all require Enterprise. Triggers that reference a resource depend on stubs + imported empty, so fill in the resource value before re-enabling the trigger. +
+ +
+ + {#if done} + + {/if} +
+ + {#if results.length} +
    + {#each results as r} +
  • + {r.ok ? '✓' : '✗'} + {r.path} + {#if !r.ok}— {r.error}{/if} +
  • + {/each} +
+ {/if} + {/if} +
+ + + + + + closeMigrationReview(false)}> + closeMigrationReview(false)}> +
+

+ This project ships migrations that recreate the data tables it uses. Review and edit the + SQL, then choose which to run. A migration runs against the data table of the same name in + {workspace}; if that data table has migrations enabled it is + recorded, otherwise it runs once as a preview job. +

+ {#each reviewList as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ {#if m.run} + + {/if} +
+ {/each} +
+ {#snippet actions()} + + + {/snippet} +
+
diff --git a/frontend/src/routes/(root)/(logged)/replay/+page.svelte b/frontend/src/routes/(root)/(logged)/replay/+page.svelte deleted file mode 100644 index d89bcbb9b0..0000000000 --- a/frontend/src/routes/(root)/(logged)/replay/+page.svelte +++ /dev/null @@ -1,79 +0,0 @@ - - -
- {#if flowRecording} -
- -
- - {:else if scriptRecording} -
- -
- - {:else} -
-
-

Replay a recording

-

- Upload a recording JSON file to replay a flow or script execution offline. -

- - Drag and drop a recording file - -
-
- {/if} -
diff --git a/frontend/src/routes/(root)/(logged)/replay/+page.ts b/frontend/src/routes/(root)/(logged)/replay/+page.ts new file mode 100644 index 0000000000..811dac8e18 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/replay/+page.ts @@ -0,0 +1,9 @@ +import { redirect } from '@sveltejs/kit' +import { base } from '$app/paths' + +// The replay page moved to /pipeline_replay (it now replays data-pipeline +// recordings in addition to flow/script ones). Redirect the old path in `load` +// so existing /replay links and bookmarks still resolve instead of 404-ing. +export function load({ url }: { url: URL }) { + redirect(307, `${base}/pipeline_replay${url.search}`) +} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 5c0f538d3f..d03fe118ee 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -605,13 +605,22 @@ } }) - onMount(() => { - let hash = page.url.hash - if (hash.startsWith('#/resource/')) { - console.log('hash', hash) - let path = hash.slice(11) - resourceEditor?.initEdit(path) + // Deep link: #/resource/ opens that resource's edit drawer. Reactive + // rather than onMount so a hash change on the already-mounted page (e.g. the + // AI session preview re-pointing its tab) opens the drawer too. Row links + // pre-set handledHash: their onclick already opens the drawer. + let handledHash = '' + $effect(() => { + const hash = page.url.hash + if (!hash.startsWith('#/resource/')) { + // Navigating away from a drawer target must clear the tracker, or + // re-targeting the same item later would be skipped as already handled. + handledHash = '' + return } + if (hash === handledHash || !resourceEditor) return + handledHash = hash + resourceEditor.initEdit(hash.slice(11)) }) let showTable = $derived( @@ -999,7 +1008,7 @@ Resource type Description - +
@@ -1014,8 +1023,17 @@ resourceEditor?.initEdit?.(path)} - >{#if marked}{@html marked}{:else}{path}{/if}{(getLocalDraftHint($workspaceStore, 'resource', path) ?? is_draft) ? '*' : ''} { + handledHash = `#/resource/${path}` + resourceEditor?.initEdit?.(path) + }} + >{#if marked}{@html marked}{:else}{path}{/if}{(getLocalDraftHint( + $workspaceStore, + 'resource', + path + ) ?? is_draft) + ? '*' + : ''} {#if draft_only} @@ -1158,83 +1176,85 @@ {/if} - - {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} - - {/if} - { - shareModal?.openDrawer?.(path, 'resource') - } - }, - { - displayName: 'Edit', - icon: Pen, - disabled: !canWrite || !showCreateButtons, - action: () => { - resourceEditor?.initEdit?.(path) - } - }, - ...(!ws_specific && isDeployable('resource', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'resource') + +
+ {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} + + {/if} + { + shareModal?.openDrawer?.(path, 'resource') + } + }, + { + displayName: 'Edit', + icon: Pen, + disabled: !canWrite || !showCreateButtons, + action: () => { + resourceEditor?.initEdit?.(path) + } + }, + ...(!ws_specific && isDeployable('resource', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'resource') + } } - } - ] - : []), - { - displayName: 'Delete', - disabled: !canWrite || !showCreateButtons, - icon: Trash, - type: 'delete', - action: (event) => { - // TODO - // @ts-ignore - if (event?.shiftKey) { - deleteResource(path, account) - } else { - deleteIsLinked = is_linked ?? false - deleteConfirmedCallback = () => { + ] + : []), + { + displayName: 'Delete', + disabled: !canWrite || !showCreateButtons, + icon: Trash, + type: 'delete', + action: (event) => { + // TODO + // @ts-ignore + if (event?.shiftKey) { deleteResource(path, account) + } else { + deleteIsLinked = is_linked ?? false + deleteConfirmedCallback = () => { + deleteResource(path, account) + } } } - } - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RotateCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadResources() + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RotateCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $workspaceStore ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadResources() + } } - } - ] - : []) - ]} - /> - + ] + : []) + ]} + /> +
{/each} {/if} @@ -1262,7 +1282,7 @@ Name Description - +
@@ -1298,7 +1318,7 @@ {removeMarkdown(truncate(description ?? '', 200))} - + {#if !canWrite} Shared globally diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index c519c01a05..e07dfcf59e 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -199,18 +199,25 @@ }) let scheduleEditor: ScheduleEditor | undefined = $state() - let hashHandled = false + // Deep link: # opens that schedule's edit drawer. Tracks the last + // handled hash (not a one-shot flag) so a hash change on the already-mounted + // page (e.g. the AI session preview re-pointing its tab) opens the drawer + // too. Row links pre-set handledHash: their onclick already opens the drawer. + let handledHash = '' $effect(() => { - if (!hashHandled && schedules.length > 0 && scheduleEditor) { - let hash = $page.url.hash - if (hash.length > 1) { - let path = hash.slice(1) - let schedule = schedules.find((s) => s.path === path) - if (schedule) { - hashHandled = true - scheduleEditor?.openEdit(path, schedule.is_flow) - } - } + const hash = $page.url.hash + if (hash.length <= 1) { + // Navigating away from a drawer target must clear the tracker, or + // re-targeting the same schedule later would be skipped as already handled. + handledHash = '' + return + } + if (hash === handledHash || schedules.length === 0 || !scheduleEditor) return + const path = hash.slice(1) + const schedule = schedules.find((s) => s.path === path) + if (schedule) { + handledHash = hash + scheduleEditor.openEdit(path, schedule.is_flow) } }) @@ -381,13 +388,22 @@ scheduleEditor?.openEdit(path, is_flow)} + onclick={() => { + handledHash = `#${path}` + scheduleEditor?.openEdit(path, is_flow) + }} class="min-w-0 grow hover:underline decoration-gray-400" >
- {summary || script_path}{(getLocalDraftHint($workspaceStore, 'trigger_schedule', path) ?? is_draft) ? '*' : ''} + {summary || script_path}{(getLocalDraftHint( + $workspaceStore, + 'trigger_schedule', + path + ) ?? is_draft) + ? '*' + : ''}
schedule: {path} diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index e9a5419b7e..4c0992395b 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -24,7 +24,6 @@ import ShareModal from '$lib/components/ShareModal.svelte' import { enterpriseLicense, - hubBaseUrlStore, userStore, userWorkspaces, workspaceStore @@ -63,7 +62,6 @@ Eye, FolderOpen, GitFork, - Globe2, History, Loader2, Pen, @@ -76,8 +74,6 @@ ChevronDown, ChevronRight } from 'lucide-svelte' - import { SCRIPT_VIEW_SHOW_PUBLISH_TO_HUB } from '$lib/consts' - import { scriptToHubUrl } from '$lib/hub' import SharedBadge from '$lib/components/SharedBadge.svelte' import Popover from '$lib/components/Popover.svelte' import ScriptVersionHistory from '$lib/components/ScriptVersionHistory.svelte' @@ -550,30 +546,6 @@ }) } - if (SCRIPT_VIEW_SHOW_PUBLISH_TO_HUB) { - menuItems.push({ - label: 'Publish to Hub', - Icon: Globe2, - onclick: () => { - if (!script) return - - window.open( - scriptToHubUrl( - script.content, - script.summary, - script.description ?? '', - script.kind, - script.language, - script.schema, - script.lock ?? '', - $hubBaseUrlStore - ).toString(), - '_blank' - ) - } - }) - } - if (showEditButtons) { if (script.archived) { menuItems.push({ diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 1e0bd25333..225bbec8ed 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -35,19 +35,26 @@ import { withWorkspaceParam } from '$lib/components/sessions/sessionMode.svelte' import { enterSessionMode } from '$lib/components/sessions/sessionSwitch.svelte' import type { SessionPreviewTabs } from '$lib/components/sessions/sessionPreviewTabs.svelte' - import { userWorkspaces, workspaceStore } from '$lib/stores' + import { userStore, userWorkspaces, usersWorkspaceStore, workspaceStore } from '$lib/stores' import { getOrCreateRuntime, getRuntime, listRuntimes } from '$lib/components/sessions/sessionRuntime.svelte' import { markSessionSeen } from '$lib/components/sessions/sessionUnread.svelte' - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { + isGlobalAiEnabled, + setSessionsBetaOptOut + } from '$lib/components/copilot/chat/global/gate' import { setToolCompletionListener } from '$lib/components/copilot/chat/shared' + import { registerToolDisplayActionHandler } from '$lib/components/copilot/chat/createdResourceActions.svelte' + import { previewTargetForSessionTarget } from '$lib/components/sessions/sessionPreviewTabs.svelte' import { base } from '$lib/base' import { + artifactKey, matchPreviewPage, pageKey, + parseArtifactRoute, parsePreviewItemRoute, previewLocationLabel, type PreviewTarget @@ -68,6 +75,41 @@ // iframe context and refuse to mount when embedded. const embedded = typeof window !== 'undefined' && window.self !== window.top + // Warm the lazily-loaded editor views (see PreviewTabHost) once the page is + // idle: entering session mode stays instant, and by the time the user opens + // an editor tab its chunk is usually already cached. Sequential so the + // prefetch trickles instead of fanning out four heavy graphs at once. + $effect(() => { + if (embedded || !globalEnabled) return + // Once the chain has started, cancelling the idle handle no longer helps — + // the disposed check between imports is what stops a user who left session + // mode from pulling the remaining graphs on whatever page they went to. + // (An import already in flight can't be aborted; only the tail is skipped.) + let disposed = false + const prefetch = async () => { + const loaders = [ + () => import('$lib/components/sessions/ScriptEditorView.svelte'), + () => import('$lib/components/sessions/FlowEditorView.svelte'), + () => import('$lib/components/sessions/RawAppEditorView.svelte'), + () => import('$lib/components/sessions/PipelineEditorView.svelte') + ] + for (const load of loaders) { + if (disposed) return + await load() + } + } + // Best-effort warming: swallow chunk-load failures — the {#await} on the + // actual open path surfaces (and retries) them. + const run = () => void prefetch().catch(() => {}) + const hasIdle = 'requestIdleCallback' in window + const handle = hasIdle ? window.requestIdleCallback(run) : window.setTimeout(run, 2000) + return () => { + disposed = true + if (hasIdle) window.cancelIdleCallback(handle) + else window.clearTimeout(handle) + } + }) + const sessionName = $derived(page.url.searchParams.get('session_name') ?? '') // Unfiltered resolution by name — drives the "session not found" fallback and @@ -93,6 +135,11 @@ // not-found UI below. $effect(() => { if (embedded || !sessionState.hydrated) return + // Family membership can't be judged before the workspace list arrives: + // workspaceRootId falls back to the raw id for workspaces it can't find, + // which makes a same-family session look foreign on a hard reload and + // would bounce the URL to another (or a brand-new) session. + if ($usersWorkspaceStore === undefined) return // sessionInCurrentFamily reads these via get(), so track them explicitly. $workspaceStore $userWorkspaces @@ -241,6 +288,10 @@ owner?.close(id) const sid = activeRuntime?.sessionId if (sid) mountedTabKeys.delete(tabKey(sid, id)) + // The active tab is excluded from the picker's pointerdown-outside (so a + // label click can toggle it); without this, closing the active tab would + // carry the open picker over to the newly active one. + activeTabPickerOpen = false } function reorderTabs(next: TabItem[]) { owner?.reorder(next.map((t) => t.id)) @@ -343,6 +394,12 @@ // Page path shown after the workspace breadcrumb — the active tab's observed // location, so the breadcrumb tracks where the user browses inside the tab. const displayPath = $derived(owner?.activeTab?.loc ?? owner?.activeTab?.url ?? `${base}/`) + // Artifacts have no workspace page, so "Open in workspace" can't resolve for them. + const activeArtifact = $derived(owner?.activeTab ? parseArtifactRoute(owner.activeTab.url) : null) + const activeTabIsArtifact = $derived(activeArtifact != null) + // The active session's artifacts, surfaced as an "Artifacts" branch in the + // preview pickers. + const sessionArtifacts = $derived(activeRuntime?.manager.artifacts.artifacts ?? []) // Writes to the tab's own session model: a hidden warm session's iframe can // finish loading while another session is shown, and its location must not // land on the visible session's tabs. @@ -406,6 +463,24 @@ } }) + // Preview cards on create/update tool calls dispatch here. Open + // (or focus, if already shown) the item's preview in the active session's panel — + // the visible chat is always the active session, so `owner` is its panel. Read + // `owner` lazily inside the handler (not in the effect body) so this registers + // once, not on every session switch. A 'focused' open leaves the tab where it is, + // so pulse it to make the click visibly land. + $effect(() => { + return registerToolDisplayActionHandler('open_item_preview', (action) => { + if (action.type !== 'open_item_preview') return + const o = owner + if (!o) return + const target = previewTargetForSessionTarget(action.previewKind, action.path) + if (!target) return + const { status } = o.open(target) + if (status === 'focused') o.pulseFocus(o.activeId) + }) + }) + // Editor-style breadcrumb over the previewed page. We only render clickable // segments when the preview is sitting on a script/flow/app route — for any // other page (home, runs, …) there's no item to drill into, so we fall back @@ -413,9 +488,13 @@ const parsedRoute = $derived(parsePreviewItemRoute(displayPath)) // Split the item path into breadcrumb dirs + leaf, mirroring EditorHeader: - // scope (`f/` | `u/`) → subfolders → item name. + // scope (`f/` | `u/`) → subfolders → item name. Prefers the + // tab's friendly path (a draft-only item's typed name): the picker tree + // groups such an item under its friendly folder, so dirs derived from the + // `…/draft_` storage path would scope the picker into a folder the + // item isn't displayed in. const segments = $derived.by(() => { - const itemPath = parsedRoute?.itemPath + const itemPath = owner?.activeTab?.friendlyPath ?? parsedRoute?.itemPath if (!itemPath) return null const parts = itemPath.split('/') if (parts.length < 3) return null @@ -456,7 +535,9 @@ ? leafKeyFor(parsedRoute.kind, parsedRoute.itemPath) : currentPage ? pageKey(currentPage.path) - : undefined + : activeArtifact + ? artifactKey(activeArtifact.id) + : undefined ) let activeTabPickerOpen = $state(false) @@ -553,10 +634,33 @@ }}>Open sessions
+ {:else if $userStore?.operator} + +
+

AI Sessions are not available for operators

+

Use the Ask AI chat instead.

+ +
{:else if !globalEnabled} -
- Sessions are gated on the global-AI dev flag. Enable with - localStorage.setItem('wm_dev_global_ai', '1') and reload. + +
+

AI Sessions are deactivated

+

You switched back to the legacy chat. Activate AI Sessions (beta) to open this page.

+
{:else if !sessionState.hydrated}
- - - + {#if !activeTabIsArtifact} + + + + {/if}
(activeTabPickerOpen = !activeTabPickerOpen)} onClose={closeTab} onReorder={reorderTabs} - class="h-8 border-b border-light bg-surface-secondary/50 {fullscreen + class="session-preview-tab-strip h-8 border-b border-light bg-surface-secondary/50 {fullscreen ? 'pl-1.5' : 'pl-9'} pr-16" > {#snippet tabAccessory(_tab, isActive)} {#if isActive} + + (e.currentTarget as HTMLElement) + .closest('[role="tab"]') + ?.focus() + }} > - {#snippet trigger()} - - {/snippet} {#snippet content()} - { - activeTabPickerOpen = false - navigatePreviewTo(t) - }} - /> + + {#key activePickerScope?.dir ?? ''} + { + activeTabPickerOpen = false + navigatePreviewTo(t) + }} + /> + {/key} {/snippet} + {/if} {/snippet} {#snippet afterTabs()} @@ -727,6 +858,7 @@ {#snippet content()} { newTabOpen = false openInNewTab(t) @@ -801,6 +933,7 @@ {#snippet content()} { emptyStateNewTabOpen = false openInNewTab(t) diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 2baf44bc77..ed01a56ac6 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -43,7 +43,7 @@ EyeOff, Circle } from 'lucide-svelte' - import { onMount, untrack } from 'svelte' + import { untrack } from 'svelte' import { page } from '$app/stores' type ListableVariableW = ListableVariable & { canWrite: boolean } @@ -244,12 +244,22 @@ }, 5000) } - onMount(() => { - let hash = $page.url.hash - if (hash.length > 1) { - let path = hash.slice(1) - variableEditor?.editVariable(path) + // Deep link: # opens that variable's edit drawer. Reactive rather than + // onMount so a hash change on the already-mounted page (e.g. the AI session + // preview re-pointing its tab) opens the drawer too. Row links pre-set + // handledHash: their onclick already opens the drawer. + let handledHash = '' + $effect(() => { + const hash = $page.url.hash + if (hash.length <= 1) { + // Navigating away from a drawer target must clear the tracker, or + // re-targeting the same item later would be skipped as already handled. + handledHash = '' + return } + if (hash === handledHash || !variableEditor) return + handledHash = hash + variableEditor.editVariable(hash.slice(1)) }) @@ -352,7 +362,7 @@ Value Description - +
@@ -366,10 +376,15 @@ variableEditor?.editVariable(path)} + onclick={() => { + handledHash = `#${path}` + variableEditor?.editVariable(path) + }} href="#{path}" > - {path}{(getLocalDraftHint($workspaceStore, 'variable', path) ?? is_draft) ? '*' : ''} + {path}{(getLocalDraftHint($workspaceStore, 'variable', path) ?? is_draft) + ? '*' + : ''} {#if draft_only} @@ -450,7 +465,11 @@
{#if refresh_error} -
+ +
- + { let owner = isOwner(path, $userStore, $workspaceStore) diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 19fa4f2553..49c29cc1a7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -15,6 +15,7 @@ import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' + import ForkMemberSettings from '$lib/components/settings/ForkMemberSettings.svelte' import SettingsPageHeader from '$lib/components/settings/SettingsPageHeader.svelte' import { WORKSPACE_SHOW_SLACK_CMD, WORKSPACE_SHOW_WEBHOOK_CLI_SYNC } from '$lib/consts' import { @@ -339,8 +340,28 @@ encryptionKeyValidationError = validation.error } }) + const currentWorkspace = $derived($userWorkspaces.find((w) => w.id === $workspaceStore)) + const canAdmin = $derived(($userStore?.is_admin ?? false) || Boolean($superadmin)) + // The creator of a fork gets the fork members screen even when they are not an admin of it: + // their `usr` row is copied from the parent, so forking as an ordinary developer leaves them + // unable to bring anyone in to collaborate. Nothing else on this page opens up — the backend + // only grants them developer memberships on the fork they created. + // The instance channels are not a valid destination on cloud or on a fork. Never select a tab + // the group does not render: saving would submit a value the API rejects, locking the whole + // error handler behind a 400. + const canUseInstanceAlerts = $derived( + !isCloudHosted() && !currentWorkspace?.parent_workspace_id + ) + const isForkOwner = $derived( + Boolean(currentWorkspace?.parent_workspace_id) && + currentWorkspace?.created_by === $userStore?.email + ) + // All state derived from URL - no local state needed let tab = $derived.by(() => { + if (!canAdmin) { + return 'users' as const + } const selectedTab = $page.url.searchParams.get('tab') as | 'users' | 'slack' @@ -594,6 +615,12 @@ initialPublicAppRateLimitPerMinute = settings.public_app_execution_limit_per_minute ?? undefined if (emptyString($enterpriseLicense)) { errorHandlerSelected = 'custom' + } else if ( + canUseInstanceAlerts && + !errorHandlerPath && + settings.error_handler_fallback_to_instance_alerts + ) { + errorHandlerSelected = 'instance_alerts' } else { errorHandlerSelected = getHandlerType(errorHandlerScriptPath) } @@ -762,8 +789,19 @@ }) $effect(() => { + // `canAdmin` is read as a dependency, not inside untrack: $userStore is repopulated + // asynchronously after a workspace switch, so the run triggered by the switch can still see + // the previous workspace's role. Re-running once it lands is what loads the settings for an + // admin who switched in from a workspace where they were not one. + const admin = canAdmin if ($workspaceStore) { untrack(() => { + // `getSettings` and the OAuth config are admin-only and carry integration secrets. A fork + // creator reaches this page for the members screen alone, which needs none of them. + if (!admin) { + loadedSettings = true + return + } loadSettings() loadSlackOAuthConfig() loadGlobalOAuthSettings() @@ -779,7 +817,8 @@ path: `${errorHandlerItemKind}/${errorHandlerScriptPath}`, extra_args: errorHandlerExtraArgs, muted_on_cancel: errorHandlerMutedOnCancel, - muted_on_user_path: errorHandlerMutedOnUserPath + muted_on_user_path: errorHandlerMutedOnUserPath, + fallback_to_instance_alerts: false } }) sendUserToast(`workspace error handler set to ${errorHandlerScriptPath}`) @@ -790,10 +829,17 @@ path: undefined, extra_args: undefined, muted_on_cancel: undefined, - muted_on_user_path: undefined + muted_on_user_path: undefined, + fallback_to_instance_alerts: errorHandlerSelected === 'instance_alerts' } }) - sendUserToast(`workspace error handler removed`) + sendUserToast( + errorHandlerSelected === 'instance_alerts' + ? `failed jobs will be reported to the instance critical alert channels` + : initialErrorHandlerScriptPath + ? `workspace error handler removed` + : `error handler settings saved` + ) } // Update initial values for dirty detection @@ -1117,13 +1163,12 @@ // The Dev workspace tab is only meaningful on a root workspace (to pair/manage a dev) or on a // dev workspace itself (to see its prod / detach). Hide it for ordinary forks — pairing isn't // available there and the backend would reject it. - const currentWsForDevTab = $derived($userWorkspaces.find((w) => w.id === $workspaceStore)) const showDevWorkspaceTab = $derived( - !currentWsForDevTab?.parent_workspace_id || (currentWsForDevTab?.is_dev_workspace ?? false) + !currentWorkspace?.parent_workspace_id || (currentWorkspace?.is_dev_workspace ?? false) ) // Navigation groups for sidebar - const navigationGroups = $derived([ + const adminNavigationGroups = $derived([ { items: [ { @@ -1300,10 +1345,29 @@ ] } ]) + + // A fork's creator manages its members through the same screen, but nothing else about the + // workspace is theirs to change, so they get the Members entry alone. + const navigationGroups = $derived( + canAdmin + ? adminNavigationGroups + : [ + { + items: [ + { + id: 'users', + label: 'Members', + aiId: 'workspace-settings-users', + aiDescription: 'Members of the fork you created' + } + ] + } + ] + ) - {#if $userStore?.is_admin || $superadmin} + {#if canAdmin || isForkOwner} {#snippet titleActions()} {#if $workspaceStore} @@ -1338,7 +1402,11 @@ {#if !loadedSettings} {:else if tab == 'users'} - + {#if canAdmin} + + {:else} + + {/if} {:else if tab == 'deploy_to'} {:else if tab == 'premium'} - {#if currentWsForDevTab?.parent_workspace_id} + {#if currentWorkspace?.parent_workspace_id} - This workspace is a fork of {currentWsForDevTab.parent_workspace_id}. It - runs on the parent's plan and its executions count toward the parent's usage and - bill, so there is no separate subscription here. Manage billing, seats, and quotas - from the parent workspace's settings. + This workspace is a fork of {currentWorkspace.parent_workspace_id}. It runs + on the parent's plan and its executions count toward the parent's usage and bill, + so there is no separate subscription here. Manage billing, seats, and quotas from + the parent workspace's settings. {:else} @@ -1761,6 +1829,7 @@ customScriptTemplate="/scripts/add?hub=hub%2F9083%2Fwindmill%2Fworkspace_error_handler_template" bind:customHandlerKind={errorHandlerItemKind} bind:handlerExtraArgs={errorHandlerExtraArgs} + showInstanceAlerts={canUseInstanceAlerts} > {#snippet customTabTooltip()} @@ -1790,24 +1859,26 @@ {/snippet} - - - - + {#if errorHandlerSelected !== 'instance_alerts'} + + + + + {/if}
{ + // Only forks are recoverable, identified two ways: the `wm-fork-` id prefix, or + // a remembered parent. Neither alone is sufficient — dev-workspace forks carry + // no prefix (only a remembered parent), while a non-member superadmin's fork has + // the prefix but no remembered parent (the recorder only sees the user's own + // membership-gated list). A workspace that is neither is left to normal loading. + const parentId = getRememberedForkParent(workspaceId) + if (parentId == undefined && !workspaceId.startsWith('wm-fork-')) return false + + // `exists` is not membership-gated, so it settles "is this workspace gone?" + // identically for members, non-members and superadmins, and it requires a valid + // session. Only a conclusive `false` means deleted: any rejection (expired + // session, transient failure) is inconclusive and must leave the normal loading + // path — including its genuine auth handling — untouched. + let exists: boolean + try { + exists = await WorkspaceService.existsWorkspace({ requestBody: { id: workspaceId } }) + } catch { + return false + } + if (exists) return false + + forgetForkParent(workspaceId) + + // Send the user to the remembered parent when we have one; otherwise (unknown or + // inaccessible parent) fall back to the picker rather than the forced-logout path + // this recovery exists to avoid. + if (parentId != undefined) { + switchWorkspace(parentId) + const parentUser = await getUserExt(parentId) + if (parentUser) { + $userStore = parentUser + sendUserToast( + `Workspace ${workspaceId} not found, switched to parent workspace ${parentId}.`, + 'warning' + ) + await goto('/') + return true + } + } + + try { + clearWorkspaceFromStorage() + } catch (e) { + console.error('Could not clear workspace storage during deleted-workspace recovery', e) + } + workspaceStore.set(undefined) + sendUserToast( + `Workspace ${workspaceId} is no longer available, please pick a workspace.`, + 'warning' + ) + await goto('/user/workspaces') + return true } async function loadUser() { @@ -41,6 +111,9 @@ await refreshSuperadmin() if ($workspaceStore) { + if (await tryRecoverFromDeletedWorkspace($workspaceStore)) { + return + } if ($userStore) { console.log(`Welcome back ${$userStore.username} to ${$workspaceStore}`) } else { @@ -131,7 +204,7 @@ computeDrift() if (page.url.pathname != '/user/login') { - setUserWorkspaceStore() + setUserWorkspaceStore().catch((e) => console.error('could not load workspace list', e)) loadUser() UserService.refreshUserToken({ ifExpiringInLessThanS: 30 * 60 }) } diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 51d093c77e..d95b78ccf3 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -29,10 +29,24 @@ if (target.tagName === 'INPUT' && (target as HTMLInputElement).type === 'number') { target.blur() } + updateEditorSwipeGuard(e) + } + + // macOS Chromium turns horizontal wheel overscroll into history navigation. Cancelling + // wheel events can't block it (only a gesture's first event is cancelable; Monaco swallows + // the rest), but Chromium honors root `overscroll-behavior-x`, so toggle it while over a + // Monaco editor. Wheel also feeds this: editors mounting under a still cursor fire no pointerover. + function updateEditorSwipeGuard(e: Event) { + const overEditor = e.target instanceof Element && e.target.closest('.monaco-editor') != null + const value = overEditor ? 'none' : '' + if (document.documentElement.style.overscrollBehaviorX !== value) { + document.documentElement.style.overscrollBehaviorX = value + document.body.style.overscrollBehaviorX = value + } } - +
{owner_name} - {#if can_write} + {#if can_write && !restricted}
import { FolderService } from '$lib/gen' import { workspaceStore, userStore } from '$lib/stores' + import { isDemoWorkspaceRestricted } from '$lib/cloud' import { ChevronDown, Pen, PlusIcon } from 'lucide-svelte' import { Button, Drawer, DrawerContent } from './common' import FolderEditor from './FolderEditor.svelte' @@ -13,6 +14,10 @@ const VALID_FOLDER_NAME = /^[a-zA-Z_0-9-]+$/ + const restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let folders: { name: string; write: boolean }[] = $state([]) let filterText: string = $state('') let selectOpen: boolean = $state(false) @@ -137,7 +142,7 @@ ) function handleSelectKeydown(e: KeyboardEvent) { - if (e.key === 'Enter' && selectOpen && noMatchingItems) { + if (e.key === 'Enter' && selectOpen && noMatchingItems && !restricted) { e.preventDefault() selectOpen = false openCreateFolder() @@ -233,18 +238,20 @@ /> {/snippet} {#snippet bottomSnippet({ close })} - + {#if !restricted} + + {/if} {/snippet}
diff --git a/frontend/src/lib/components/ForkWorkspaceBanner.svelte b/frontend/src/lib/components/ForkWorkspaceBanner.svelte index e2bfae895e..b97740b135 100644 --- a/frontend/src/lib/components/ForkWorkspaceBanner.svelte +++ b/frontend/src/lib/components/ForkWorkspaceBanner.svelte @@ -1,7 +1,8 @@ + +{#if approved} + {@render children()} +{:else} + +
+

+ Rendering it can expose you to XSS attacks. Only enable it if you trust the author of this app. +

+ +
+
+{/if} diff --git a/frontend/src/lib/components/MoveDrawer.svelte b/frontend/src/lib/components/MoveDrawer.svelte index 212e45d58d..21b155fce3 100644 --- a/frontend/src/lib/components/MoveDrawer.svelte +++ b/frontend/src/lib/components/MoveDrawer.svelte @@ -48,6 +48,7 @@ push('schedule', c.schedule_count) push('kafka', c.kafka_count) push('mqtt', c.mqtt_count) + push('amqp', c.amqp_count) push('nats', c.nats_count) push('postgres', c.postgres_count) push('sqs', c.sqs_count) diff --git a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte index e7651e5703..70a182ea62 100644 --- a/frontend/src/lib/components/ParqetCsvTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte @@ -26,6 +26,10 @@ // through the app-scoped, provenance-gated `apps_u/*` endpoints instead of // the viewer-scoped `job_helpers/*` API. Undefined in the editor/preview. appPath?: string | undefined + // HMAC bearer (`exp=..&sig=..`) minted by `signS3Objects`. When present on a + // deployed-app read, it bypasses the provenance gate — matching the download + // and image routes. Forwarded to every app-scoped preview/count/export call. + presigned?: string | undefined } let { @@ -33,9 +37,18 @@ storage, workspaceId, disable_download = false, - appPath = undefined + appPath = undefined, + presigned = undefined }: Props = $props() + // Split the presigned bearer into typed query params for the generated client. + // Only meaningful in deployed-app (`appPath`) reads; empty otherwise. + function presignedParams(): { sig?: string; exp?: string } { + if (!appPath || !presigned) return {} + const p = new URLSearchParams(presigned) + return { sig: p.get('sig') ?? undefined, exp: p.get('exp') ?? undefined } + } + // Route the parquet/csv read through the app-scoped endpoints when `appPath` // is set, else the viewer-scoped helpers. Same request/response shape either // way — the only difference is which identity authorizes the S3 read. @@ -48,7 +61,8 @@ fileKey: s3resource, searchCol, searchTerm, - storage + storage, + ...presignedParams() }) : HelpersService.loadTableRowCount({ workspace, @@ -71,7 +85,14 @@ const workspace = workspaceId ?? $workspaceStore! const csv = s3resource.endsWith('.csv') if (appPath) { - const data = { workspace, path: appPath, fileKey: s3resource, storage, ...args } + const data = { + workspace, + path: appPath, + fileKey: s3resource, + storage, + ...args, + ...presignedParams() + } return csv ? AppService.appLoadCsvPreview(data) : AppService.appLoadParquetPreview(data) } const data = { workspace, path: s3resource, storage, ...args } @@ -230,7 +251,7 @@ {/if} {#if !disable_download && !s3resource.endsWith('.csv')} {@const csvApiPath = appPath - ? `/w/${workspaceId}/apps_u/download_s3_parquet_file_as_csv/${appPath}?file_key=${encodeURIComponent(s3resource)}${storage ? `&storage=${storage}` : ''}` + ? `/w/${workspaceId}/apps_u/download_s3_parquet_file_as_csv/${appPath}?file_key=${encodeURIComponent(s3resource)}${storage ? `&storage=${storage}` : ''}${presigned ? `&${presigned}` : ''}` : `/w/${workspaceId}/job_helpers/download_s3_parquet_file_as_csv?file_key=${encodeURIComponent(s3resource)}${storage ? `&storage=${storage}` : ''}`} {@const csvName = (s3resource.split('/').pop() ?? 'download') + '.csv'} {#if shouldDownloadViaClient()} diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index 27d6b45e6c..284677ce7c 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -20,6 +20,7 @@ PostgresTriggerService, NatsTriggerService, MqttTriggerService, + AmqpTriggerService, SqsTriggerService, GcpTriggerService, AzureTriggerService, @@ -58,6 +59,7 @@ | 'postgres_trigger' | 'nats_trigger' | 'mqtt_trigger' + | 'amqp_trigger' | 'sqs_trigger' | 'gcp_trigger' | 'azure_trigger' @@ -302,6 +304,11 @@ workspace: ws!, path: path }) + } else if (kind === 'amqp_trigger') { + return await AmqpTriggerService.existsAmqpTrigger({ + workspace: ws!, + path: path + }) } else if (kind == 'sqs_trigger') { return await SqsTriggerService.existsSqsTrigger({ workspace: ws!, diff --git a/frontend/src/lib/components/ResourceForm.svelte b/frontend/src/lib/components/ResourceForm.svelte index ef226501b7..2c563a8e95 100644 --- a/frontend/src/lib/components/ResourceForm.svelte +++ b/frontend/src/lib/components/ResourceForm.svelte @@ -296,7 +296,7 @@

Resource type '{resource_type}' not found in your workspace

- onLoadResourceType?.()} /> + onLoadResourceType?.()} />

Define the value in JSON directly

{/if} diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index e74e6f7285..60b15277e0 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -331,6 +331,7 @@ class="mt-1" _resourceMetadata={{ resource_type: resourceType }} asset={{ kind: 'resource', path: value }} + workspace={effectiveWorkspace} /> {/if} diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index 9b447a7d0c..ed93550669 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -565,7 +565,7 @@ {:else}
-
+

-
+
- - + + {#snippet extra()} {#if warnJobLimit} {warnJobLimitMsg} diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 8829660ea3..9f702a53c3 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -38,13 +38,17 @@ onSelectAndClose }: Props = $props() - let ws = $derived(workspace ?? $workspaceStore) - let drawer: Drawer | undefined = $state() let s3FilePickerInner: S3FilePickerInner | undefined = $state() let workspaceSettingsInitialized = $state(true) let storage: string | undefined = $state(undefined) + let s3ResourcePath: string | undefined = $state(undefined) + /** Per-open workspace override, for callers (e.g. the global explorer) whose + * asset lives in a different workspace than this picker was mounted for. */ + let workspaceOverride: string | undefined = $state(undefined) + let effectiveWorkspace = $derived(workspaceOverride ?? workspace) + let ws = $derived(effectiveWorkspace ?? $workspaceStore) let uploadModalOpen = $state(false) let allFilesByKey: Record< @@ -66,8 +70,15 @@ { lazy: true } ) - export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { - secondaryStorageNames.refetch() + export async function open( + _preSelectedFileKey: S3Object | undefined = undefined, + opts: { s3ResourcePath?: string; workspace?: string } = {} + ) { + s3ResourcePath = opts.s3ResourcePath + workspaceOverride = opts.workspace + if (!s3ResourcePath) { + secondaryStorageNames.refetch() + } drawer?.openDrawer?.() await tick() @@ -86,12 +97,14 @@ size="1200px" > { s3FilePickerInner?.exit?.() drawer?.closeDrawer?.() }} - tooltip="Files present in the Workspace S3 bucket. You can set the workspace S3 bucket in the settings." + tooltip={s3ResourcePath + ? `Files present in the bucket of the ${s3ResourcePath} resource.` + : 'Files present in the Workspace S3 bucket. You can set the workspace S3 bucket in the settings.'} documentationLink="https://www.windmill.dev/docs/integrations/s3" > {#snippet actions()}
- {#if secondaryStorageNames.current?.length} + {#if !s3ResourcePath && secondaryStorageNames.current?.length} +
+ + + +
+ {/if} + + +
+ {#snippet header()} + + Cache the results for each possible inputs + + {/snippet} +
+ !!script.cache_ttl, (v) => (script.cache_ttl = v ? 300 : undefined)} + options={{ right: 'Cache the results for each possible inputs' }} + /> + {#if script.cache_ttl} +
How long to keep the cache valid
+ + script.cache_ignore_s3_path, (v) => (script.cache_ignore_s3_path = v || undefined) + } + options={{ + right: 'Ignore S3 Object paths for caching purposes', + rightTooltip: + 'If two S3 objects passed as input have the same content, they will hit the same cache entry, regardless of their path.' + }} + /> + {/if} +
+
+ +
+ {#snippet header()} + + Add a custom timeout for this script + + {/snippet} +
+ { + if (script.timeout && script.timeout != undefined) { + script.timeout = undefined + } else { + script.timeout = 300 + } + }} + options={{ right: 'Add a custom timeout for this script' }} + /> + {#if Boolean(script.timeout)} + Timeout duration + + {/if} +
+
+ +
+ {#snippet header()} + + Debounce Jobs + + {/snippet} + +
+ +
+ {#snippet header()} + + Restart the script upon ending unless cancelled + + {/snippet} + { + script.restart_unless_cancelled = script.restart_unless_cancelled ? undefined : true + }} + options={{ right: 'Restart upon ending unless cancelled' }} + /> +
+ +
+ {#snippet header()} + + In this mode, the script is meant to be run on dedicated workers that run the script at + native speed. Can reach >1500rps per dedicated worker. Only available on enterprise + edition and for Python3, Deno, Bun and Bunnative. + + {/snippet} + { + script.dedicated_worker = script.dedicated_worker ? undefined : true + }} + options={{ right: 'Script is run on dedicated workers' }} + /> + {#if script.dedicated_worker} +
+ + A worker group needs to be configured to listen to this script. Select it in the dedicated + workers section of the worker group configuration. + +
+ {/if} +
+ +
+ {#snippet header()} + + The logs, arguments and results of the job will be completely deleted from Windmill after + the specified delay once it is complete. Set to 0 for immediate deletion. The deletion is + irreversible. This settings ONLY applies when the script is used within a flow or triggered + synchronously. + {#if !$enterpriseLicense} + This option is only available on Windmill Enterprise Edition. + {/if} + + {/snippet} +
+ { + script.delete_after_secs = script.delete_after_secs != null ? undefined : 0 + }} + options={{ right: 'Delete logs, arguments and results after completion' }} + /> + {#if script.delete_after_secs != null} + + {/if} +
+
+ + {#if !isCloudHosted()} +
+ {#snippet header()} + + Jobs from script labeled as high priority take precedence over the other jobs when in the + jobs queue. + {#if !$enterpriseLicense}This is a feature only available on enterprise edition.{/if} + + {/snippet} + 0} + on:change={() => { + script.priority = script.priority ? undefined : 100 + }} + options={{ right: 'Label as high priority' }} + > + {#snippet right()} + { + if (script.priority && script.priority > 100) { + script.priority = 100 + } else if (script.priority && script.priority < 0) { + script.priority = 0 + } + }} + /> + {/snippet} + +
+ {/if} +
diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index d7214e9a24..bd30cd6909 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -87,6 +87,7 @@ import DefaultScripts from './DefaultScripts.svelte' import { getContext, onMount, setContext, tick, untrack } from 'svelte' import EditorHeader from './EditorHeader.svelte' + import ScriptSettingsBadges from './ScriptSettingsBadges.svelte' import AutosaveIndicator from './AutosaveIndicator.svelte' import LabelsInput from './LabelsInput.svelte' @@ -331,7 +332,7 @@ triggersState }) - const enterpriseLangs = ['bigquery', 'snowflake', 'mssql', 'oracledb'] + const enterpriseLangs = ['mssql', 'oracledb'] // Languages the pipeline editor treats as warehouse/dataset transforms — // the ones where a `-- pipeline` annotation is a natural next step. @@ -1971,12 +1972,26 @@ {onOpenOthersDrafts} /> {/if} + {#if !condensedHeader} + {@const canOpenRuntime = + customUi?.topBar?.settings != false && + customUi?.settingsPanel?.disableRuntime !== true} + { + selectedTab = 'runtime' + metadataOpen = true + } + : undefined} + /> + {/if}
- {#if $enterpriseLicense && initialPath != ''} + {#if $enterpriseLicense && initialPath != '' && !inSessionPane} {/if} diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 803684f49e..ccc3f04be7 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -30,6 +30,10 @@ allowEdit?: boolean allowView?: boolean clearable?: boolean + /** Workspace to list runnables from. Defaults to the navigation + * `$workspaceStore`; pass the session's acting workspace so a forked + * session lists its own scripts/flows/apps rather than the parent's. */ + workspace?: string } let { @@ -42,9 +46,15 @@ allowRefresh = false, allowEdit = true, allowView = true, - clearable = false + clearable = false, + workspace = undefined }: Props = $props() + let effectiveWorkspace = $derived(workspace ?? $workspaceStore) + // Only carry the workspace onto Edit/View routes when an explicit override + // was passed, so existing callers' links are unchanged. + let wsParam = $derived(workspace ? `?workspace=${encodeURIComponent(workspace)}` : '') + let items: { value: string; label: string }[] = $state([]) let drawerViewer: Drawer | undefined = $state() let drawerFlowViewer: Drawer | undefined = $state() @@ -58,7 +68,7 @@ async function loadItems(): Promise { if (itemKind == 'flow') { items = ( - await FlowService.listFlows({ workspace: $workspaceStore!, withoutDescription: true }) + await FlowService.listFlows({ workspace: effectiveWorkspace!, withoutDescription: true }) ).map((flow) => ({ value: flow.path, label: `${flow.path}${flow.summary ? ` | ${truncate(flow.summary, 20)}` : ''}`, @@ -67,7 +77,7 @@ } else if (itemKind == 'script') { items = ( await ScriptService.listScripts({ - workspace: $workspaceStore!, + workspace: effectiveWorkspace!, kinds: kinds.join(','), withoutDescription: true }) @@ -76,7 +86,7 @@ label: `${script.path}${script.summary ? ` | ${truncate(script.summary, 20)}` : ''}` })) } else if (itemKind == 'app') { - items = (await AppService.listApps({ workspace: $workspaceStore! })).map((app) => ({ + items = (await AppService.listApps({ workspace: effectiveWorkspace! })).map((app) => ({ value: app.path, label: `${app.path}${app.summary ? ` | ${truncate(app.summary, 20)}` : ''}` })) @@ -84,7 +94,7 @@ } $effect(() => { - itemKind && $workspaceStore && untrack(() => loadItems()) + itemKind && effectiveWorkspace && untrack(() => loadItems()) }) let darkMode: boolean = $state(false) @@ -99,7 +109,7 @@ - + @@ -158,7 +168,7 @@ target="_blank" variant="default" size="xs" - href="{base}/flows/edit/{scriptPath}">EditEdit {/if} {#if allowView} @@ -181,7 +191,7 @@ target="_blank" variant="default" size="xs" - href="{base}/apps/edit/{scriptPath}" + href="{base}/apps/edit/{scriptPath}{wsParam}" > Edit @@ -192,7 +202,7 @@ size="xs" target="_blank" startIcon={{ icon: Code }} - href="{base}/apps/get/{scriptPath}" + href="{base}/apps/get/{scriptPath}{wsParam}" > View @@ -206,7 +216,7 @@ target="_blank" variant="default" size="xs" - href="{base}/scripts/edit/{scriptPath}" + href="{base}/scripts/edit/{scriptPath}{wsParam}" > Edit @@ -217,7 +227,10 @@ size="xs" startIcon={{ icon: Code }} on:click={async () => { - const { language, content } = await getScriptByPath(scriptPath ?? '') + const { language, content } = await getScriptByPath( + scriptPath ?? '', + effectiveWorkspace + ) code = content lang = language drawerViewer?.openDrawer() diff --git a/frontend/src/lib/components/ScriptSettingsBadges.svelte b/frontend/src/lib/components/ScriptSettingsBadges.svelte new file mode 100644 index 0000000000..8d73609891 --- /dev/null +++ b/frontend/src/lib/components/ScriptSettingsBadges.svelte @@ -0,0 +1,42 @@ + + +{#if badges.length > 0} +
+ {#each badges as badge (badge.key)} + + + onclick?.(badge.key) : undefined} + aria-label={`${badge.label}: ${badge.detail}`} + /> + {#snippet text()} + {badge.label} — {badge.detail} + {/snippet} + + {/each} +
+{/if} diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 0996e6ca7b..29da96cf98 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -21,9 +21,14 @@ import { safeSelectItems } from './select/utils.svelte' import Toggle from './Toggle.svelte' import { Trash } from 'lucide-svelte' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' const dispatch = createEventDispatcher() + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + type Kind = | 'script' | 'group_' @@ -42,6 +47,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'azure_trigger' + | 'amqp_trigger' | 'email_trigger' | 'volume' let kind: Kind @@ -268,7 +274,9 @@ > {/if}
- {#if own} + {#if own && restricted} + {DEMO_RESTRICTION_HINT} + {:else if own}
(owner = '')}> @@ -310,7 +318,7 @@

{owner} {#if own} + >{#if own && !restricted}
- {:else}{write}{/if}
{#if own} diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index 5ca069b5be..3763f7aeed 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -16,7 +16,10 @@ type Props = { input: DbInput - onData: (data: Record[]) => void + /** `ranCode` is the editor content that produced `data`, captured when the + * run started, so a caller can discard a late response whose query it has + * moved past. */ + onData: (data: Record[], ranCode: string) => void placeholderTableName?: string /** Called after a migration is run via the DDL guard, so the schema view * can be refreshed to reflect the applied change. */ @@ -46,6 +49,12 @@ const DEFAULT_SQL = 'SELECT * FROM _' let code = $state(DEFAULT_SQL) + + /** Seed the editor from outside, e.g. a query composed in a drawer. */ + export function setCode(newCode: string) { + code = newCode + editor?.setCode?.(newCode) + } $effect(() => { const _code = untrack(() => code) if (placeholderTableName && _code === DEFAULT_SQL) { @@ -71,6 +80,12 @@ if (pruneComments(code).trim() === '') return } + // Snapshot the code that will actually run: after the DDL guard has stripped + // any migrated statements, and before the execution await during which + // `code` can be re-seeded (setCode). The caller identifies which query a late + // response belongs to by this value, and history restores exactly it. + const ranCode = code + isRunning = true try { const statements = splitSqlStatements(pruneComments(code)) @@ -130,10 +145,13 @@ created_by: '', id: job.id, success: true, - code, + // The snapshot, not the live `code`: a mid-run setCode() must not + // rebind this entry's result to a different query. Selecting it later + // reseeds the editor with this exact SQL and re-fires onData with it. + code: ranCode, result }) - onData(result) + onData(result, ranCode) } if (doPostgresRowToJsonFix) sendUserToast('Query failed but recovered with the row_to_json fix') @@ -185,7 +203,7 @@ on:select={(e) => { const data = e.detail as (typeof runHistory)[number] editor?.setCode(data.code) - onData(data.result) + onData(data.result, data.code) }} /> diff --git a/frontend/src/lib/components/SyncResourceTypes.svelte b/frontend/src/lib/components/SyncResourceTypes.svelte index dd343e43f6..acda8c0567 100644 --- a/frontend/src/lib/components/SyncResourceTypes.svelte +++ b/frontend/src/lib/components/SyncResourceTypes.svelte @@ -6,13 +6,19 @@ interface Props { onSynced?: () => void + // When set, the endpoint returns an explicit not-found error if the hub does + // not know this type (the sync itself still refreshes the whole list). + resourceType?: string } - let { onSynced = undefined }: Props = $props() + let { onSynced = undefined, resourceType = undefined }: Props = $props() let hubRtSync = usePromise( async () => { - const res = await fetch('/api/settings/sync_cached_resource_types', { method: 'POST' }) + const url = resourceType + ? `/api/settings/sync_cached_resource_types?name=${encodeURIComponent(resourceType)}` + : '/api/settings/sync_cached_resource_types' + const res = await fetch(url, { method: 'POST' }) if (!res.ok) { const body = await res.text() throw new Error(body || res.statusText) diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 7c59a2a324..37e3e152db 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -3,15 +3,16 @@ import { Database, Loader2 } from 'lucide-svelte' import Button from './common/button/Button.svelte' + import Tooltip from './meltComponents/Tooltip.svelte' import { sendUserToast } from '$lib/toast' import { workspaceStore } from '$lib/stores' import { tryEvery } from '$lib/utils' interface Props { - workspaceOverride?: string | undefined; - resourceType: string | undefined; - args?: Record | any; - buttonTextOverride?: string | undefined; + workspaceOverride?: string | undefined + resourceType: string | undefined + args?: Record | any + buttonTextOverride?: string | undefined } let { @@ -19,13 +20,15 @@ resourceType, args = {}, buttonTextOverride = undefined - }: Props = $props(); + }: Props = $props() const scripts: { [key: string]: { code: string lang: string argName: string + // Shown as an info tooltip next to the button, e.g. to clarify where the test executes + tooltip?: string additionalCheck?: (testResult: CompletedJob) => CompletedJob } } = { @@ -97,7 +100,9 @@ export async function main(s3: S3) { } `, lang: 'bun', - argName: 's3' + argName: 's3', + tooltip: + 'The storage operations of this test run on the Windmill server (the API process), not on the worker. If no access key/secret key is set, the ambient AWS credentials of the server (environment variables, instance role) are used — scripts using this resource directly through an S3 SDK resolve credentials on the worker instead, so results may differ.' }, azure_blob: { code: ` @@ -125,7 +130,9 @@ export async function main(s3: S3) { } `, lang: 'bun', - argName: 's3' + argName: 's3', + tooltip: + 'The storage operations of this test run on the Windmill server (the API process), not on the worker.' }, graphql: { code: '{ __typename }', @@ -171,7 +178,9 @@ export async function main(bucket: any) { } `, lang: 'bun', - argName: 'bucket' + argName: 'bucket', + tooltip: + "The storage operations of this test run on the Windmill server (the API process). If no credentials are configured, the server's ambient credentials for the configured provider (environment variables, instance role) are used." } } @@ -236,13 +245,20 @@ export async function main(bucket: any) { } -{#if Object.keys(scripts).includes(resourceType || '')} - + {#if scripts[resourceType].tooltip} + + {#snippet text()}{scripts[resourceType].tooltip}{/snippet} + {/if} - {buttonTextOverride ?? 'Test connection'} - + {/if} diff --git a/frontend/src/lib/components/WorkspaceDeployLayout.svelte b/frontend/src/lib/components/WorkspaceDeployLayout.svelte index b7d87d6321..491d649a44 100644 --- a/frontend/src/lib/components/WorkspaceDeployLayout.svelte +++ b/frontend/src/lib/components/WorkspaceDeployLayout.svelte @@ -34,6 +34,7 @@ deploymentStatus: Record allSelected?: boolean emptyMessage?: string + hideSelection?: boolean children?: Snippet // Snippets for customization @@ -64,6 +65,7 @@ deploymentStatus, allSelected = false, emptyMessage = 'No items to deploy', + hideSelection = false, header, alerts, selectAllActions, @@ -150,12 +152,13 @@ {@render alerts()} {/if} - + {#if items.length > 0 || selectAllActions}
- {#if items.length > 0} + {#if items.length > 0 && !hideSelection}
{/if} - {#if $enterpriseLicense && $appPath != ''} + {#if $enterpriseLicense && $appPath != '' && !inSessionPane}
diff --git a/frontend/src/lib/components/apps/editor/PublicApp.svelte b/frontend/src/lib/components/apps/editor/PublicApp.svelte index 92cde8788b..8995684e28 100644 --- a/frontend/src/lib/components/apps/editor/PublicApp.svelte +++ b/frontend/src/lib/components/apps/editor/PublicApp.svelte @@ -51,8 +51,9 @@ // Use workspace from props or from app.workspace_id (for custom path responses) let effectiveWorkspace = $derived(workspace ?? app?.workspace_id) - // HTML results from runnables only need viewer approval on the public - // surfaces (untrusted distribution); the in-workspace viewer never gated them. + // On the public surfaces (untrusted distribution) runnable-authored html/svg needs + // the viewer's approval before it renders, unless the app sandbox isolates it. The + // in-workspace viewer renders it verbatim. See getAppMarkupTrust. setContext(IS_APP_PUBLIC_CONTEXT_KEY, !inWorkspace) // WIN-2006: inside the opaque viewer iframe, navigations to other routes diff --git a/frontend/src/lib/components/apps/markupTrust.ts b/frontend/src/lib/components/apps/markupTrust.ts new file mode 100644 index 0000000000..2cabcfe065 --- /dev/null +++ b/frontend/src/lib/components/apps/markupTrust.ts @@ -0,0 +1,42 @@ +import { BROWSER } from 'esm-env' +import { getContext } from 'svelte' +import { IS_APP_PUBLIC_CONTEXT_KEY } from './types' + +/** + * How a component should treat runnable-authored `html`/`svg` markup. + * + * - `sanitize`: strip scripts/handlers first. The default everywhere outside an app + * (job results, previews, flows), where the markup's author and its viewer are + * different users. + * - `trusted`: render verbatim, so dynamic svg/html keeps working. + * - `approval`: render verbatim, but only after the viewer opts in. + */ +export type MarkupTrust = 'sanitize' | 'trusted' | 'approval' + +/** + * True when this document has an opaque origin, i.e. it is the app sandbox's iframe + * and markup rendered here cannot reach the viewer's Windmill session. + * + * Read the real origin rather than a `wm_embed`/framing signal: those are set by + * whoever framed us, so a hostile page could claim to be the sandbox while running + * on the Windmill origin with the viewer's cookies. The origin can't be faked the + * same way — anyone who makes it opaque has, by doing so, given up the session + * access this gate exists to protect. + */ +function isOpaqueOrigin(): boolean { + return BROWSER && window.origin === 'null' +} + +/** + * Trust level for markup rendered by a low-code app component. Call during init. + * + * An app renders its own author's markup, so it is `trusted` — except on the public + * surfaces, where the app is distributed to untrusted viewers. There it needs the + * viewer's approval, unless the app is sandbox-isolated, in which case the markup + * can't reach the viewer's session and needs no gate. + */ +export function getAppMarkupTrust(): MarkupTrust { + const isPublic = getContext(IS_APP_PUBLIC_CONTEXT_KEY) + if (!isPublic) return 'trusted' + return isOpaqueOrigin() ? 'trusted' : 'approval' +} diff --git a/frontend/src/lib/components/apps/migrateApp.ts b/frontend/src/lib/components/apps/migrateApp.ts index 7e6d4f12cb..323d020d60 100644 --- a/frontend/src/lib/components/apps/migrateApp.ts +++ b/frontend/src/lib/components/apps/migrateApp.ts @@ -4,10 +4,11 @@ import { allItems } from './editor/appUtilsCore' /** * Normalize an `App` in place to the current schema: default `hiddenInlineScripts` - * type, migrate the legacy `doNotRecomputeOnInputChanged` flag, and default - * `fullHeight` on every grid item. Lives in its own light module (no app-editor - * component imports) so non-editor callers — e.g. the localStorage→DB draft - * migration — can reuse it without pulling the whole `apps/utils` graph. + * type, migrate the legacy `doNotRecomputeOnInputChanged` flag, default a missing + * `grid`, and default `fullHeight` on every grid item. Lives in its own light + * module (no app-editor component imports) so non-editor callers, e.g. the + * localStorage→DB draft migration, can reuse it without pulling the whole + * `apps/utils` graph. */ export function migrateApp(app: App) { ;(app?.hiddenInlineScripts ?? []).forEach((x) => { @@ -22,6 +23,14 @@ export function migrateApp(app: App) { } }) + // A stored app value can have no `grid` at all: a persisted draft row is the + // confirmed case, and the editor renders a draft in place of the deployed + // value. The grid components dereference it unguarded, so normalize it here, + // the one hook every load path runs through. + if (!Array.isArray(app.grid)) { + app.grid = [] + } + allItems(app.grid, app.subgrids).forEach((x) => { gridColumns.forEach((column: number) => { if (x?.[column]?.fullHeight === undefined) { diff --git a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte index 4745742e20..880abb685a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte @@ -94,6 +94,12 @@ description: 'Triggered by an MQTT message', icon: Radio }, + { + id: 'amqp', + label: 'On AMQP', + description: 'Triggered by an AMQP (RabbitMQ) message', + icon: Radio + }, { id: 'nats', label: 'On NATS', diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 64bd2790cb..3e5e9d8cef 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -171,6 +171,19 @@ * initial viewport fit re-arms when it changes, so switching folders * in-place gets a fresh fit. */ viewportFitKey?: string + /** Tint the currently-running runnable nodes amber (the compute is + * happening now). Opt-in — the recorder's replay player turns it on so + * the active transform stands out; the live editor stays calm. */ + highlightActiveRun?: boolean + /** Asset node id (`asset:${kind}:${path}`) → a monotonic "recompute" nonce. + * When a node's nonce changes, it flashes a fading green background — its + * producer just recomputed it. Driven by the replay player frame-by-frame. */ + recomputedAssetIds?: ReadonlyMap + /** Let the wheel zoom the canvas (and swallow the page scroll while doing + * so). Default true for the full-height editor/player. Set false when the + * canvas is embedded inline inside a scrollable container, so a wheel + * gesture over it scrolls the container instead of being captured. */ + scrollZoom?: boolean } let { graph, @@ -199,7 +212,10 @@ boundPick, onPickEnd, showMinimap = true, - viewportFitKey = '' + viewportFitKey = '', + highlightActiveRun = false, + recomputedAssetIds, + scrollZoom = true }: Props = $props() // `${kind}:${path}` ids for the hovered / pinned runs (both script and flow @@ -385,7 +401,10 @@ producers: producersByAsset.get(`${a.kind}:${a.path}`) ?? [], onRunProducer, dataTestGuarded, - producerFailed + producerFailed, + // Bumped by the replay player when this asset's producer just + // recomputed it — the node flashes green and fades. + recomputePulse: recomputedAssetIds?.get(assetId) } }) } @@ -474,6 +493,9 @@ downstreamCount: downstreamByScript.get(r.path) ?? 0, downstreamUnsavedCount: downstreamUnsavedByScript.get(r.path) ?? 0, runState, + // Amber-tint this node while it's the transform actively running + // (replay player only — the live editor keeps its calm styling). + highlightRunning: highlightActiveRun, // Bounded-cascade entrypoint: only valid starts (schedule / // manual roots) with downstream get the "Run downstream up // to…" menu item. @@ -1172,6 +1194,8 @@ nodesDraggable={false} nodesConnectable={false} elementsSelectable + zoomOnScroll={scrollZoom} + preventScrolling={scrollZoom} zoomOnDoubleClick={false} connectionLineType={ConnectionLineType.SmoothStep} defaultEdgeOptions={{ type: 'asset' }} diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 425565353a..d48ecbe09e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -493,6 +493,10 @@ // restores its input. Guarded on the path so a staging round-trip // (emit → page → runFormInitialArgs) doesn't re-seed and loop. The read-only // branch uses PipelineScriptView's own onArgsChange instead. + // Declared before the pre-effect that seeds it: a `$state` referenced by an + // earlier-registered `$effect.pre` hits a TDZ ("Cannot access 'args' before + // initialization") when the pane remounts and the pre-effect runs before this + // line executes. let args = $state>({}) let argsSeedPath: string | undefined = undefined $effect.pre(() => { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte index 762bdbc0fc..999559a0c7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte @@ -74,6 +74,10 @@ // asset failed. Escalates the guard badge from "protected" to a // failed-run outcome (rolled-back on EE, published-anyway on CE). producerFailed?: boolean + // Monotonic nonce bumped by the replay player when this asset's + // producer just recomputed it. A change triggers a one-shot green + // fade so a freshly-written table stands out as the run progresses. + recomputePulse?: number } // SvelteFlow injects this on the node component when the user clicks // the node. Combined with our own `hovered` state to drive the @@ -170,6 +174,13 @@ onmouseleave={() => (hovered = false)} role="presentation" > + {#if data.recomputePulse !== undefined} + {#key data.recomputePulse} + +
+ {/key} + {/if}
+ + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte index 3022063c4e..e364b29883 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte @@ -8,6 +8,7 @@ GcpTriggerService, KafkaTriggerService, MqttTriggerService, + AmqpTriggerService, NatsTriggerService, PostgresTriggerService, ScheduleService, @@ -15,6 +16,7 @@ } from '$lib/gen' import KafkaTriggerEditor from '$lib/components/triggers/kafka/KafkaTriggerEditor.svelte' import MqttTriggerEditor from '$lib/components/triggers/mqtt/MqttTriggerEditor.svelte' + import AmqpTriggerEditor from '$lib/components/triggers/amqp/AmqpTriggerEditor.svelte' import NatsTriggerEditor from '$lib/components/triggers/nats/NatsTriggerEditor.svelte' import PostgresTriggerEditor from '$lib/components/triggers/postgres/PostgresTriggerEditor.svelte' import SqsTriggerEditor from '$lib/components/triggers/sqs/SqsTriggerEditor.svelte' @@ -22,6 +24,7 @@ import EmailTriggerEditor from '$lib/components/triggers/email/EmailTriggerEditor.svelte' import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' import WebhookEditor from '$lib/components/triggers/webhook/WebhookEditor.svelte' + import { setTriggerWorkspace } from '$lib/components/triggers/triggerWorkspace' // Owns the native-trigger drawer wiring for the pipeline canvas: the nine // editor instances, the create/edit dispatch by kind, and the delete @@ -34,11 +37,16 @@ // (matching the previous `{#if mode === 'edit'}` wrapper). The webhook // editor stays mounted in every mode — its node is clickable in view mode // too (informational endpoint URLs/token). - type Props = { onUpdate: () => void; mountTriggerEditors: boolean } - let { onUpdate, mountTriggerEditors }: Props = $props() + type Props = { onUpdate: () => void; mountTriggerEditors: boolean; workspace?: string } + let { onUpdate, mountTriggerEditors, workspace: triggerWorkspace }: Props = $props() + + // Register the trigger-workspace resolver for the whole editor subtree (the + // nine editors + the delete handler below). See triggerWorkspace.ts. + setTriggerWorkspace(() => triggerWorkspace ?? $workspaceStore) let kafkaEditor: KafkaTriggerEditor | undefined = $state() let mqttEditor: MqttTriggerEditor | undefined = $state() + let amqpEditor: AmqpTriggerEditor | undefined = $state() let natsEditor: NatsTriggerEditor | undefined = $state() let postgresEditor: PostgresTriggerEditor | undefined = $state() let sqsEditor: SqsTriggerEditor | undefined = $state() @@ -58,6 +66,8 @@ return kafkaEditor?.openNew(false, scriptPath) case 'mqtt': return mqttEditor?.openNew(false, scriptPath) + case 'amqp': + return amqpEditor?.openNew(false, scriptPath) case 'nats': return natsEditor?.openNew(false, scriptPath) case 'postgres': @@ -82,6 +92,8 @@ return kafkaEditor?.openEdit(triggerPath, false, scriptPath) case 'mqtt': return mqttEditor?.openEdit(triggerPath, false, scriptPath) + case 'amqp': + return amqpEditor?.openEdit(triggerPath, false, scriptPath) case 'nats': return natsEditor?.openEdit(triggerPath, false, scriptPath) case 'postgres': @@ -115,9 +127,9 @@ } async function confirmDeleteAttachedTrigger() { - if (!triggerDeleteTarget || !$workspaceStore) return + const workspace = triggerWorkspace ?? $workspaceStore + if (!triggerDeleteTarget || !workspace) return const { kind, path: triggerPath } = triggerDeleteTarget - const workspace = $workspaceStore triggerDeleteLoading = true try { switch (kind) { @@ -130,6 +142,9 @@ case 'mqtt': await MqttTriggerService.deleteMqttTrigger({ workspace, path: triggerPath }) break + case 'amqp': + await AmqpTriggerService.deleteAmqpTrigger({ workspace, path: triggerPath }) + break case 'nats': await NatsTriggerService.deleteNatsTrigger({ workspace, path: triggerPath }) break @@ -190,6 +205,7 @@ gated off the canvas outside edit mode. --> + diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index 4cb65ee8f3..d8a977539b 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -48,6 +48,9 @@ // Last-run status + run count observed this session (from the // folder queue poll). Undefined until the first observed run. runState?: RunnableRunState + // Opt-in (replay player): when this node is the transform actively + // running, tint its whole surface amber so the compute is unmissable. + highlightRunning?: boolean // True for nodes synthesized from local drafts (script not yet // persisted). Same convention as `unsaved` on triggers/edges. unsaved?: boolean @@ -103,6 +106,9 @@ let hover = $state(false) let menuOpen = $state(false) let running = $state(false) + + // Amber "computing now" surface, gated so only the replay player lights it up. + let computingNow = $derived(data.highlightRunning === true && data.runState?.status === 'running') // Popover state for the on-node Run-button caret. Sticky while open so // `showRun` (which gates the whole pill) stays true even after the // pointer leaves the node — otherwise picking an option would unmount @@ -195,7 +201,9 @@ 'flex items-center rounded-md drop-shadow-sm overflow-hidden border transition-colors', 'bg-surface border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500', selected && 'bg-surface-accent-selected border-border-selected', - data.unsaved && 'border-2 border-dashed border-gray-400 dark:border-gray-500' + data.unsaved && 'border-2 border-dashed border-gray-400 dark:border-gray-500', + computingNow && + 'bg-amber-50 dark:bg-amber-900/30 border-amber-400 dark:border-amber-600 animate-pulse' )} style="width: {NODE.width}px; min-height: {NODE.height}px;" title={nodeTooltip} diff --git a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte index 98307013c4..0e279112cd 100644 --- a/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte @@ -48,6 +48,7 @@ email: { icon: Mail, label: 'email', ...MUTED }, kafka: { icon: Zap, label: 'kafka', ...MUTED }, mqtt: { icon: Radio, label: 'mqtt', ...MUTED }, + amqp: { icon: Radio, label: 'amqp', ...MUTED }, nats: { icon: MessageSquare, label: 'nats', ...MUTED }, postgres: { icon: Database, label: 'postgres', ...MUTED }, sqs: { icon: Send, label: 'sqs', ...MUTED }, diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts index e4692de6b8..89c9f24708 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.test.ts @@ -509,15 +509,13 @@ describe('assetUriToNodeId', () => { expect(assetUriToNodeId('ducklake://lake/t')).toBe('ducklake:lake/t') expect(assetUriToNodeId('not-a-uri')).toBeUndefined() }) - it('strips leading slashes from S3 keys so s3:/// and s3:// share a node', () => { - // Mirror of Rust `parse_asset_syntax`: `--to s3:///exports/x` must resolve - // to the same canonical node as the graph's `s3object:exports/x`. - expect(assetUriToNodeId('s3:///exports/x')).toBe('s3object:exports/x') - expect(assetUriToNodeId('s3:///exports/x')).toBe(assetUriToNodeId('s3://exports/x')) - // All leading slashes are stripped so a canonical key never starts with - // `/` (the quad-slash `S3Object(s3="/x")` form collapses to `x`). - expect(assetUriToNodeId('s3:////x')).toBe('s3object:x') + it('keeps the S3 storage distinction (verbatim suffix)', () => { + // Mirror of Rust `parse_asset_syntax`: the suffix is kept verbatim, so a + // default-storage `s3:///exports/x` resolves to `s3object:/exports/x` + // while `s3://exports/x` names storage `exports` — a different node. + expect(assetUriToNodeId('s3:///exports/x')).toBe('s3object:/exports/x') + expect(assetUriToNodeId('s3://exports/x')).toBe('s3object:exports/x') // Hive-partition keys and non-S3 kinds are untouched. - expect(assetUriToNodeId('s3:///t/y=2024/f.parquet')).toBe('s3object:t/y=2024/f.parquet') + expect(assetUriToNodeId('s3:///t/y=2024/f.parquet')).toBe('s3object:/t/y=2024/f.parquet') }) }) diff --git a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts index 6fcabbe71f..f3ed9a4f66 100644 --- a/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts +++ b/frontend/src/lib/components/assets/AssetGraph/boundedCascade.ts @@ -38,11 +38,10 @@ export function assetUriToNodeId(uri: string): string | undefined { // `s3` is the URI prefix for the `s3object` asset kind (mirrors the CLI // `assetUri` and the canvas). All other kinds use their name verbatim. const kind = prefix === 's3' ? 's3object' : prefix - // Mirror Rust `parse_asset_syntax`: strip all leading slashes from S3 keys so - // `s3:///key` (default storage) and `s3://key` resolve to the same node id - // and a canonical key never starts with `/`. - const path = kind === 's3object' ? m[2].replace(/^\/+/, '') : m[2] - return `${kind}:${path}` + // The suffix is kept verbatim (mirrors Rust `parse_asset_syntax`): an S3 + // path encodes the storage, with a leading `/` for the workspace default + // (`s3:///key` → `/key`) vs `s3://secondary/key` → `secondary/key`. + return `${kind}:${m[2]}` } // Native trigger kinds that fan out *per event*: a single event always flows @@ -51,6 +50,7 @@ export function assetUriToNodeId(uri: string): string | undefined { const EVENT_TRIGGER_KINDS: ReadonlySet = new Set([ 'kafka', 'mqtt', + 'amqp', 'nats', 'postgres', 'sqs', diff --git a/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts index ede66ebe4b..407dc886f9 100644 --- a/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts +++ b/frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts @@ -20,6 +20,11 @@ import type { AssetGraphResponse } from './types' export const CASCADE_POLL_INTERVAL_MS = 1000 export const CASCADE_JOB_TIMEOUT_MS = 30 * 60 * 1000 +// Data-asset kinds a pipeline graph resolves — the `asset_kinds` filter for the +// `/assets/graph` fetch. Shared so the pipeline editor and deploy-to-hub request +// the same nodes/edges (and can't silently diverge when a kind is added). +export const DATA_ASSET_KINDS = ['s3object', 'ducklake', 'datatable', 'volume'] + export type LocalScriptContent = { content: string language: Preview['language'] diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index f9e53494ff..5fdbc4c748 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -31,6 +31,7 @@ const NATIVE_TRIGGER_KEYWORDS: NativeTriggerKind[] = [ 'email', 'kafka', 'mqtt', + 'amqp', 'nats', 'postgres', 'sqs', @@ -276,18 +277,11 @@ function stripTrailingKvOpts(s: string): string { function parseAssetSyntax(s: string): PipelineTriggerAsset | undefined { for (const [prefix, kind] of ASSET_PREFIXES) { if (s.startsWith(prefix)) { - let path = s.slice(prefix.length) - // Mirror the Rust `parse_asset_syntax` S3 canonicalization: strip all - // leading slashes so the SDK object form (`s3:///key`, default - // storage) and DuckDB / `// on s3://key` share one asset path, and a - // canonical key never starts with `/` (so ref reconstruction - // round-trips). Without this the live graph preview would show - // disconnected `/key` and `key` nodes. S3-only; leading slashes only, - // so Hive-partition keys are untouched. - if (kind === 's3object') { - path = path.replace(/^\/+/, '') - } - return { kind, path } + // The suffix is kept verbatim, mirroring the Rust `parse_asset_syntax`. + // For S3 the path encodes the storage: `s3:///key` yields `/key` + // (default storage, leading slash significant) while + // `s3://secondary/key` yields `secondary/key` — two different objects. + return { kind, path: s.slice(prefix.length) } } } return undefined diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts index a1a82870c8..e5f3b74e3d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.test.ts @@ -114,6 +114,37 @@ describe('pipeline AI direct-draft helpers', () => { expect(drafts().size).toBe(0) }) + // The output_kind seed is a random placeholder that live inference overwrites + // and deploy re-derives, so it must not read as detected lineage — else a + // dynamic/unwritten output looks wired when the deployed script has no edge. + it('does not report the output_kind seed as detected lineage', async () => { + vi.spyOn(ScriptService, 'getScriptByPath').mockRejectedValue(new Error('404')) + const { handle, drafts } = makeHandle() + const res = await handle.proposeNode({ + path: 'f/x/seeded', + language: 'duckdb' as any, + content: '-- pipeline\n-- on schedule\nSELECT 1', + outputKind: 'ducklake' as any + }) + expect(drafts().get('f/x/seeded')?.outputAssets?.length).toBeGreaterThan(0) + expect(res.detectedWrites).toEqual([]) + }) + + // `inferAssets` returns the `// materialize` target separately from body + // writes, so it must be folded into detectedWrites or a canonical materialize + // node would falsely read as having no output. + it('reports the `-- materialize` target as a detected write', async () => { + vi.spyOn(ScriptService, 'getScriptByPath').mockRejectedValue(new Error('404')) + const { handle } = makeHandle() + const res = await handle.proposeNode({ + path: 'f/x/mat', + language: 'duckdb' as any, + content: '-- pipeline\n-- on schedule\n-- materialize ducklake://main/out\nSELECT 1', + outputKind: 'ducklake' as any + }) + expect(res.detectedWrites).toEqual(['ducklake://main/out']) + }) + it('editNode rejects a path outside the open folder', async () => { const { handle, drafts } = makeHandle() await expect(handle.editNode('f/other/foo', '-- pipeline')).rejects.toThrow(/open folder/) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts index 3658bc5a32..a6874d9b64 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineAiHelpers.ts @@ -7,7 +7,7 @@ import { type AssetWithAltAccessType } from '$lib/components/assets/lib' import { assetUri, autoOutputAsset, type PipelineOutputKind } from './pipelineTemplates' -import { parsePipelineAnnotations } from './parsePipelineAnnotations' +import { parsePipelineAnnotations, scd2CurrentTargetPath } from './parsePipelineAnnotations' import type { AssetGraphResponse } from './types' import type { PipelineAIChatHelpers, @@ -107,6 +107,31 @@ async function inferDraftAssets( } } +// `inferAssets` returns body reads/writes but NOT the `// materialize` target, +// which the parser surfaces separately (resolveGraph adds it the same way). +// A managed materialize node's output is real and deployable, so fold its +// target(s) into the detected writes. +function materializeWrites(content: string): Array<{ kind: AssetKind; path: string }> { + const m = parsePipelineAnnotations(content).materialize + if (!m) return [] + const out = [{ kind: m.targetKind, path: m.targetPath }] + const current = scd2CurrentTargetPath(m) + if (current) out.push({ kind: m.targetKind, path: current }) + return out +} + +function dedupeAssets( + assets: Array<{ kind: AssetKind; path: string }> +): Array<{ kind: AssetKind; path: string }> { + const seen = new Set() + return assets.filter((a) => { + const key = `${a.kind}:${a.path}` + if (seen.has(key)) return false + seen.add(key) + return true + }) +} + export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIChatHelpers { // A staged draft is always persisted into the OPEN folder's data_pipeline // bundle, so a path outside the folder would silently land an unrelated script @@ -240,17 +265,30 @@ export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIC (outputKind ? autoOutputAsset(outputKind as PipelineOutputKind, deps.getFolder(), language) : undefined) + // Effective outputs = what actually becomes an output edge on the canvas: + // body/annotation-inferred writes, or the output_kind seed as a fallback. + const outputAssets = + inferred.writes.length > 0 ? inferred.writes : seeded ? [seeded] : undefined const next = new Map(drafts) next.set(path, { localId: deps.newDraftLocalId(), script: makePipelineScript(language, path, content, new Date().toISOString()), - outputAssets: inferred.writes.length > 0 ? inferred.writes : seeded ? [seeded] : undefined, + outputAssets, inputAssets: inferred.reads }) deps.setDrafts(next) deps.onShowDrafts?.() deps.onProposeNode?.(path) - return { path } + // Report deployable lineage only (body writes + `// materialize` target), + // never the random `seeded` placeholder — else a dynamic/unwritten output + // reads as wired when the deployed script has no such edge. + return { + path, + detectedReads: inferred.reads.map(assetUri), + detectedWrites: dedupeAssets([...inferred.writes, ...materializeWrites(content)]).map( + assetUri + ) + } }, editNode: async (path, content) => { deps.ensureEditable?.() @@ -273,16 +311,24 @@ export function createPipelineAiHelpers(deps: PipelineAiHelperDeps): PipelineAIC baseScript = await ScriptService.getScriptByPath({ workspace, path }) } const inferred = await inferDraftAssets(baseScript.language, content) + const outputAssets = inferred.writes.length > 0 ? inferred.writes : existing?.outputAssets const next = new Map(drafts) next.set(path, { localId: existing?.localId ?? deps.newDraftLocalId(), script: { ...baseScript, content }, - outputAssets: inferred.writes.length > 0 ? inferred.writes : existing?.outputAssets, + outputAssets, inputAssets: inferred.reads }) deps.setDrafts(next) deps.onShowDrafts?.() deps.onProposeNode?.(path) + // Deployable lineage only (see proposeNode): body writes + materialize target. + return { + detectedReads: inferred.reads.map(assetUri), + detectedWrites: dedupeAssets([...inferred.writes, ...materializeWrites(content)]).map( + assetUri + ) + } }, removeProposedNode: async (path) => { if (!deps.getDrafts().has(path)) { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts index 00fdd085ed..08ec089491 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.test.ts @@ -9,16 +9,15 @@ import { // The seeded draft asset (`autoOutputAsset`, stored as `outputAssets` and used // by resolveGraph for inactive-draft node identity) must match the asset // identity the deploy-time / wasm parser infers from the generated body. The -// parser canonicalizes any S3 URI by stripping the `s3://` prefix and all -// leading slashes (see backend `parse_asset_syntax`); if the seed carried a -// leading slash while the body wrote `s3:///key`, the preview would render a -// duplicate `/key` node and a phantom post-deploy drift. This pins the two in -// lockstep so that class of drift can't regress. +// parser keeps the suffix after `s3://` verbatim (see backend +// `parse_asset_syntax`), so a default-storage object's path carries a leading +// slash (`s3:///key` → `/key`). If the seed and the body's write URI disagree, +// the preview renders a duplicate node and a phantom post-deploy drift. This +// pins the two in lockstep so that class of drift can't regress. -// Mirror of the parser's S3 canonicalization for a raw `s3://…` URI. +// Mirror of the parser's S3 path extraction for a raw `s3://…` URI. function canonicalS3Key(uri: string): string { - const rest = uri.replace(/^s3:\/\//, '') - return rest.replace(/^\/+/, '') + return uri.replace(/^s3:\/\//, '') } const S3_KINDS: PipelineOutputKind[] = ['s3_parquet', 's3_object'] @@ -32,10 +31,10 @@ describe('pipelineTemplates S3 seed/body parity', () => { expect(output).toBeDefined() const asset = output! - // The seed must be a canonical slashless key so it matches the - // identity the parser infers from the generated body. + // The seed must carry the default-storage leading slash so it + // matches the identity the parser infers from the generated body. expect(asset.kind).toBe('s3object') - expect(asset.path.startsWith('/')).toBe(false) + expect(asset.path.startsWith('/')).toBe(true) const body = generatePipelineDraft({ language, diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index 02607d5f35..1e26f2bb31 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -175,16 +175,16 @@ export function autoOutputAsset( case 'ducklake': case 'materialize': return { kind: 'ducklake', path: `main/${adj}_${pick(TABLE_NOUNS)}_${slug}` } - // s3 outputs use the canonical slashless key. `parse_asset_syntax` - // normalizes `s3:///` (default storage) and `s3://` to the - // bare ``, so the seeded draft asset must be slashless to match the - // deploy-time inferred identity — otherwise the post-deploy drift check - // would flag the output as a phantom `/`-prefixed node. The generated - // bodies still emit the `s3:///` default-storage URI for runtime I/O. + // s3 paths carry the canonical leading slash of a default-storage + // object (`s3:///` parses to path `/`). The deploy-time + // parser stores writes in that form — a slashless seeded path would + // never match it, and the post-deploy drift check would report the + // output as lost (it isn't; the key differs by one '/'). Bodies emit + // the path verbatim after `s3://`, so the slash round-trips. case 's3_parquet': return { kind: 's3object', - path: `pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` + path: `/pipelines/${folder}/${adj}_${pick(DATASET_NOUNS)}_${slug}.parquet` } case 's3_object': { // duckdb's natural output for a generic blob is CSV (one COPY TO @@ -194,7 +194,7 @@ export function autoOutputAsset( const ext = language === 'duckdb' ? 'csv' : 'json' return { kind: 's3object', - path: `pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` + path: `/pipelines/${folder}/${adj}_${pick(FILE_NOUNS)}_${slug}.${ext}` } } // A macro library produces no asset — its "output" is the registry @@ -222,13 +222,6 @@ export function assetUri(asset: { kind: AssetKind; path: string }): string { return `${ASSET_URI_PREFIX[asset.kind]}${asset.path}` } -// Bare object key for the SDK's `{ s3: }` / `s3:///` forms. Asset -// paths are already canonical slashless keys; strip stray leading slashes -// defensively so the emitted key never starts with '/'. -function s3Key(path: string): string { - return path.replace(/^\/+/, '') -} - // Splits a datatable asset path (`/` or `/.
`) // into its constituent parts. The `.
` grammar is owned by // `parseDbInputFromAssetSyntax` in $lib/utils.ts (which consumes a full @@ -309,6 +302,7 @@ export type DraftTriggerSource = | 'email' | 'kafka' | 'mqtt' + | 'amqp' | 'nats' | 'postgres' | 'sqs' @@ -437,11 +431,12 @@ function bodyTs(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': - // `s3:///` URI — one spelling shared with the `// on - // s3:///…` annotation form (the object literal `{ s3: }` - // is equivalent). + // `input.path` encodes storage as `/` (an empty + // storage segment — leading slash — is the workspace default). + // Emit it verbatim after `s3://` so a named-storage input keeps + // its storage; stripping the slash reads the default-storage key. return [ - ` const buf = await wmill.loadS3File(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, + ` const buf = await wmill.loadS3File(${JSON.stringify(`s3://${input.path}`)})`, ` const rows = JSON.parse(new TextDecoder().decode(buf))`, `` ].join('\n') @@ -467,10 +462,10 @@ function bodyTs(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': - // `s3:///` URI — see the loadS3File note above. + // `s3:///` URI — see the loadS3File note above. return [ ` const payload = new TextEncoder().encode(JSON.stringify(rows))`, - ` await wmill.writeS3File(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, payload)` + ` await wmill.writeS3File(${JSON.stringify(`s3://${output.path}`)}, payload)` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -524,11 +519,12 @@ function bodyPython(ctx: TemplateContext): string { if (!input) return '' switch (input.kind) { case 's3object': - // `s3:///` URI — SDK string params must be s3:// URIs - // (bare keys are rejected), and this form matches the - // `# on s3:///…` annotation spelling. + // SDK string params must be s3:// URIs (bare keys are rejected). + // `input.path` encodes storage as `/` (empty storage + // segment — leading slash — is the workspace default), so emit it + // verbatim after `s3://` to preserve a named-storage input. return [ - ` buf = wmill.load_s3_file(${JSON.stringify(`s3:///${s3Key(input.path)}`)})`, + ` buf = wmill.load_s3_file(${JSON.stringify(`s3://${input.path}`)})`, ` import json; rows = json.loads(buf.decode("utf-8"))` ].join('\n') case 'datatable': @@ -551,10 +547,10 @@ function bodyPython(ctx: TemplateContext): string { switch (outputKind) { case 's3_parquet': case 's3_object': - // `s3:///` URI — see the load_s3_file note above. + // `s3:///` URI — see the load_s3_file note above. return [ ` import json`, - ` wmill.write_s3_file(${JSON.stringify(`s3:///${s3Key(output.path)}`)}, json.dumps(rows).encode("utf-8"))` + ` wmill.write_s3_file(${JSON.stringify(`s3://${output.path}`)}, json.dumps(rows).encode("utf-8"))` ].join('\n') case 'datatable': { const dbName = output.path.split('/')[0] ?? 'main' @@ -646,7 +642,7 @@ function bodyDuckdb(ctx: TemplateContext): string { if (!input) return null switch (input.kind) { case 's3object': - return `read_parquet('s3:///${input.path}')` + return `read_parquet('s3://${input.path}')` case 'datatable': // `pg` is the attached Postgres catalog (see ATTACH above). // Use a 2-part `pg.
` ref so the asset parser maps it @@ -669,7 +665,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3:///${output.path}' (FORMAT 'parquet');` + `) TO 's3://${output.path}' (FORMAT 'parquet');` ) } break @@ -680,7 +676,7 @@ function bodyDuckdb(ctx: TemplateContext): string { `COPY (`, ` SELECT *`, ` FROM ${fromExpr}`, - `) TO 's3:///${output.path}' (FORMAT 'csv', HEADER);` + `) TO 's3://${output.path}' (FORMAT 'csv', HEADER);` ) } break diff --git a/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts index 4a1de04e13..56a82715fc 100644 --- a/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts +++ b/frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts @@ -408,6 +408,11 @@ export function mapWarningsToMarkers(code: string, warnings: ContractWarning[]): (l) => /^\s*(\/\/|--|#)\s*column\s/.test(l) && !!w.column && l.includes(w.column) ) break + // The measure/dimension warning kinds (`missing_measure_column`, + // `missing_dimension_column`, `non_aggregate_measure`) are not handled + // here: this live mirror does not parse metric annotations, so it never + // produces them. They are surfaced by the authoritative post-deploy + // `check_schema_contracts` endpoint, not by Monaco markers. case 'missing_relationship_column': case 'relationship_type_mismatch': lineNumber = lineMatching( diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 0e055abca2..d9ef2dafbf 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -110,6 +110,7 @@ export type NativeTriggerKind = | 'email' | 'kafka' | 'mqtt' + | 'amqp' | 'nats' | 'postgres' | 'sqs' diff --git a/frontend/src/lib/components/chat/ChatMessage.svelte b/frontend/src/lib/components/chat/ChatMessage.svelte index f811abd0a2..181ca68700 100644 --- a/frontend/src/lib/components/chat/ChatMessage.svelte +++ b/frontend/src/lib/components/chat/ChatMessage.svelte @@ -6,6 +6,7 @@ import LinkRenderer from '$lib/components/copilot/chat/LinkRenderer.svelte' import DisplayResult from '$lib/components/DisplayResult.svelte' import { workspaceStore } from '$lib/stores' + import { markdownProse } from '$lib/components/markdownProse' interface Props { role: 'user' | 'assistant' | 'tool' | 'system' @@ -103,7 +104,7 @@ {/if} {/if} -
+
({ label: name, value: i + 1 }))} />
{:else}
+ {#if aiChatManager.mode === AIMode.GLOBAL} + + {/if} {#if !hideModeSelector} {/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 8f39b78a27..336246b368 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -3,7 +3,13 @@ import ContextElementBadge from './ContextElementBadge.svelte' import ContextTextarea from './ContextTextarea.svelte' import autosize from '$lib/autosize' - import type { ContextElement } from './context' + import { + contextElementKey, + createAttachedFileContextElement, + isSameContextElement, + type AppDomSelectorElement, + type ContextElement + } from './context' import { AIMode } from './AIChatManager.svelte' import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' import { formatMention } from './mention' @@ -11,11 +17,33 @@ import { tick, untrack, type Snippet } from 'svelte' import Portal from '$lib/components/Portal.svelte' import { zIndexes } from '$lib/zIndexes' - import { ArrowUp, Square } from 'lucide-svelte' + import { ArrowUp, Loader2, Square, X } from 'lucide-svelte' import { Button } from '$lib/components/common' import { sendUserToast } from '$lib/toast' import { type PasteAttachment } from './pasteTokens' import { chatDraft, expanded } from './chatDraft' + import { + fileToAttachedImage, + isImageFile, + MAX_ATTACHED_IMAGES, + MAX_IMAGE_BYTES, + type AttachedImage + } from './imageUtils' + import { modelSupportsVision } from '../modelConfig' + import { tryGetCurrentModel } from '$lib/aiStore' + import { createLongHash } from '$lib/editorLangUtils' + import { + fileToAttachedTextFile, + MAX_ATTACHED_FILES, + MAX_CONVERSATION_FILE_BYTES, + MAX_TEXT_FILE_BYTES, + textByteLength, + type AttachedTextFile + } from './textFileUtils' + import { MessageDraft } from './messageDraft.svelte' + import ExpandableImage, { + isImageViewerOpen + } from '$lib/components/common/image/ExpandableImage.svelte' const aiChatManager = getAiChatManager() @@ -27,6 +55,8 @@ placeholder?: string initialInstructions?: string initialPastes?: PasteAttachment[] + initialImages?: AttachedImage[] + initialFiles?: AttachedTextFile[] editingMessageIndex?: number | null onEditEnd?: () => void className?: string @@ -56,6 +86,8 @@ placeholder, initialInstructions = '', initialPastes = undefined, + initialImages = undefined, + initialFiles = undefined, editingMessageIndex = null, onEditEnd = () => {}, className = '', @@ -122,13 +154,223 @@ let contextTextareaComponent: ContextTextarea | undefined = $state() let instructionsTextareaComponent: HTMLTextAreaElement | undefined = $state() - let instructions = $state(untrack(() => initialInstructions)) + // The four lanes that ship with the next send — text, collapsed big-paste + // blobs, per-message images, per-message text files — owned by one draft so + // every aggregation applies the draft rules. The composer keeps only the + // async in-flight accounting (pending counters, byte reservations). + const draft = new MessageDraft( + untrack(() => ({ + text: initialInstructions, + pastes: initialPastes ?? [], + images: initialImages ?? [], + files: initialFiles ?? [] + })) + ) $effect(() => { - const text = instructions + const text = draft.text untrack(() => onDraftChange?.(text)) }) - // Collapsed big-paste blobs referenced by tokens in `instructions`. - let pastes = $state(untrack(() => initialPastes ?? [])) + // Images being decoded right now. Holds off sending so a message can never go + // out without an attachment the user already dropped, and reserves cap slots + // against a concurrent drop. + let pendingImages = $state(0) + + /** Attach dropped/pasted image files (downscaled + bounded). GLOBAL mode only. */ + export async function addImages(files: (File | Blob)[]) { + if (aiChatManager.mode !== AIMode.GLOBAL) return + const imageFiles = files.filter(isImageFile) + if (imageFiles.length === 0) return + // tryGetCurrentModel returns undefined instead of throwing: this runs from a + // drop/paste handler that can't surface a rejection. + const model = tryGetCurrentModel() + // Only known text-only models fail this, so attaching would certainly 400 the + // next turn — refuse rather than warn and send it anyway. + if (model && !modelSupportsVision(model.provider, model.model)) { + sendUserToast(`${model.model} can't read images. Switch to a vision model first.`, true) + return + } + // Count decodes already in flight: two drops that both read the image count + // before either resolves would each claim the same free slots and overshoot + // the cap. + const remaining = MAX_ATTACHED_IMAGES - draft.images.length - pendingImages + if (remaining <= 0) { + sendUserToast(`You can attach up to ${MAX_ATTACHED_IMAGES} images.`, true) + return + } + const oversized = imageFiles.filter((f) => f.size > MAX_IMAGE_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_IMAGE_BYTES / 1_000_000) + sendUserToast(`${oversized.length} image(s) over ${mb}MB were skipped.`, true) + } + const usable = imageFiles.filter((f) => f.size <= MAX_IMAGE_BYTES) + if (usable.length === 0) return + const batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${usable.length - batch.length} were skipped.`, + true + ) + } + // Claim the slots before awaiting, and hold sending until they resolve: + // decoding takes ~50-800ms, and a send during it would clear `images` while + // this closure still appends to it, landing the picture on the next message. + pendingImages += batch.length + try { + // One at a time: a decoded bitmap costs ~4 bytes per pixel (a 12MP photo is + // ~48MB), so decoding the whole batch at once would hold every one of them + // live simultaneously. + const added: AttachedImage[] = [] + let failed = 0 + for (const file of batch) { + try { + added.push(await fileToAttachedImage(file)) + } catch { + failed++ + } + } + if (added.length > 0) draft.addImages(added) + if (failed > 0) sendUserToast(`Could not attach ${failed} image(s).`, true) + } finally { + pendingImages -= batch.length + } + } + + function removeImage(index: number) { + draft.images = draft.images.filter((_, i) => i !== index) + } + + // Files being read right now — same send-hold/slot-reservation role as pendingImages. + let pendingFiles = $state(0) + // Drop routing resolves file-system handles/entries asynchronously before it + // can call addTextFiles/addImages; a send during that window would land the + // dropped files on the NEXT message. Holds block sending (no slot or chip + // impact) until the drop handler finishes routing. + let ingestionHolds = $state(0) + export function holdSendForIngestion(): () => void { + ingestionHolds += 1 + let released = false + return () => { + if (!released) { + released = true + ingestionHolds -= 1 + } + } + } + // Bytes those in-flight reads have claimed against the conversation budget: + // two overlapping drops that both read the budget before either lands would + // otherwise each spend the same remaining allowance. + let pendingFileBytes = $state(0) + + // Publish this composer's staged bytes (committed attachments + in-flight + // reads) to the manager so a concurrently-mounted composer — the edit box + // while editing an earlier message — sees them in its own budget check and + // the two can't each spend the whole conversation allowance. + const composerKey = untrack(() => createLongHash()) + let stagedBytes = $derived( + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) + pendingFileBytes + ) + $effect(() => { + aiChatManager.setComposerStaged(composerKey, editingMessageIndex, stagedBytes) + }) + $effect(() => () => aiChatManager.clearComposerStaged(composerKey)) + + /** Attach dropped/picked text files (sniffed + bounded). GLOBAL mode only. */ + export async function addTextFiles(candidates: File[]) { + if (aiChatManager.mode !== AIMode.GLOBAL) return + if (candidates.length === 0) return + const remaining = MAX_ATTACHED_FILES - draft.files.length - pendingFiles + if (remaining <= 0) { + sendUserToast(`You can attach up to ${MAX_ATTACHED_FILES} files.`, true) + return + } + const oversized = candidates.filter((f) => f.size > MAX_TEXT_FILE_BYTES) + if (oversized.length > 0) { + const mb = Math.round(MAX_TEXT_FILE_BYTES / 1_000_000) + sendUserToast( + `${oversized.length} file(s) over ${mb}MB were skipped — link their folder to read them on demand.`, + true + ) + } + const usable = candidates.filter((f) => f.size <= MAX_TEXT_FILE_BYTES) + if (usable.length === 0) return + let batch = usable.slice(0, remaining) + if (batch.length < usable.length) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_FILES} files; ${usable.length - batch.length} were skipped.`, + true + ) + } + // Conversation-level byte budget: transcript + queue + every live + // composer's stage (this one and, mid-edit, the other) + this composer's + // own pending reads. File content is persisted with every history save, so + // an unbounded total would grow the chat record without limit. The + // transcript sum skips any message a composer is editing — that composer's + // stage stands in for it, so counting both would charge those bytes twice. + let budget = + MAX_CONVERSATION_FILE_BYTES - + aiChatManager.attachmentBytesExcluding(composerKey) - + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - + pendingFileBytes + const withinBudget: File[] = [] + for (const f of batch) { + if (f.size <= budget) { + withinBudget.push(f) + budget -= f.size + } + } + if (withinBudget.length < batch.length) { + const mb = Math.round(MAX_CONVERSATION_FILE_BYTES / 1_000_000) + sendUserToast( + `${batch.length - withinBudget.length} file(s) skipped — this conversation reached its ${mb}MB attachment budget. Link a folder to read larger sets on demand.`, + true + ) + } + batch = withinBudget + if (batch.length === 0) return + pendingFiles += batch.length + const reservedBytes = batch.reduce((sum, f) => sum + f.size, 0) + pendingFileBytes += reservedBytes + try { + const reads: { name: string; content: string }[] = [] + let skipped = 0 + for (const file of batch) { + try { + const attached = await fileToAttachedTextFile(file) + if (attached) reads.push(attached) + else skipped++ + } catch { + skipped++ + } + } + // Commit through the draft in one synchronous step — fold (dedupe, + // courtesy rename) and decoded-byte admission both run against the live + // list, so another batch landing between this one's file reads can't be + // missed, and malformed input that inflates on decode can't slip past the + // raw-size admission above. This batch's own raw reservation is excluded + // from the budget — the decoded sizes replace it. + const liveBudget = + MAX_CONVERSATION_FILE_BYTES - + aiChatManager.attachmentBytesExcluding(composerKey) - + draft.files.reduce((sum, f) => sum + textByteLength(f.content), 0) - + (pendingFileBytes - reservedBytes) + const { droppedAtBudget } = draft.addFiles(reads, liveBudget) + if (droppedAtBudget > 0) { + const mb = Math.round(MAX_CONVERSATION_FILE_BYTES / 1_000_000) + sendUserToast( + `${droppedAtBudget} file(s) skipped — this conversation reached its ${mb}MB attachment budget. Link a folder to read larger sets on demand.`, + true + ) + } + if (skipped > 0) sendUserToast(`Skipped ${skipped} file(s) (non-text).`, true) + } finally { + pendingFiles -= batch.length + pendingFileBytes -= reservedBytes + } + } + + function removeFile(index: number) { + draft.files = draft.files.filter((_, i) => i !== index) + } // App mode @ mention state let showAppContextTooltip = $state(false) @@ -145,6 +387,12 @@ aiChatManager.mode === AIMode.GLOBAL ) + const domSelectorChips = $derived( + (selectedContext ?? []).filter((c): c is AppDomSelectorElement => c.type === 'app_dom_selector') + ) + + const contextKey = contextElementKey + /** Append `@title` to the textarea so the button-picker path stays in * sync with the inline `@` mention path — both leave a visible * token tied to the selectedContext entry, which the textarea diffs on @@ -153,9 +401,9 @@ * leave duplicate tokens. */ export function insertMention(title: string) { const target = `@${title}` - if (instructions.split(/\s+/).includes(target)) return - const sep = instructions.length === 0 || /\s$/.test(instructions) ? '' : ' ' - instructions = `${instructions}${sep}${target} ` + if (draft.text.split(/\s+/).includes(target)) return + const sep = draft.text.length === 0 || /\s$/.test(draft.text) ? '' : ' ' + draft.text = `${draft.text}${sep}${target} ` } /** Strip every `@title` token from the textarea — used when the user @@ -171,7 +419,7 @@ contextTextareaComponent?.unsyncMention(title) const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') const re = new RegExp(`(^|\\s)@${escaped}(\\s|$)`, 'g') - instructions = instructions.replace(re, (_m, lead, trail) => { + draft.text = draft.text.replace(re, (_m, lead, trail) => { // Boundary on at least one side → drop the mention entirely. if (!lead || !trail) return '' // Middle of text: keep ONE of the bracketing whitespace chars so @@ -190,31 +438,83 @@ } // Restore composer contents after a rolled-back turn. No-op when the user - // already typed a new draft — restoring would clobber it. - export function restoreInstructions(value: string, restoredPastes: PasteAttachment[] = []) { - if (instructions.trim()) return - instructions = value - pastes = restoredPastes + // already drafted something new — typed text or attached images (including + // ones still decoding) — restoring would clobber it. + /** Returns whether the restore was taken: an occupied composer keeps its own + * draft and declines, and the caller must then leave that draft's context + * alone too — restoring context for text that was dropped would retarget the + * draft the user is still writing. */ + export function restoreInstructions( + value: string, + restoredPastes: PasteAttachment[] = [], + restoredImages: AttachedImage[] = [], + restoredFiles: AttachedTextFile[] = [] + ): boolean { + // Attachments still decoding/reading (or mid-drop-routing) count as + // occupancy too — they belong to a draft the user started even though + // their lane is still empty. + if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) return false + if ( + !draft.replaceIfEmpty({ + text: value, + pastes: restoredPastes, + images: restoredImages, + files: restoredFiles + }) + ) { + return false + } focusInput() + return true } /** Put text back into the textarea (queued-message delete, or restore * after a cancelled/errored turn), prepended to any draft so nothing - * the user typed is lost. */ - export function prependText(text: string) { - instructions = instructions.trim() ? `${text}\n\n${instructions}` : text + * the user typed is lost. Restored images join whatever is already + * attached, up to the cap — dropping them would lose the attachment + * silently, which is the whole reason the queue carries them. */ + export function prependText( + text: string, + restoredImages: AttachedImage[] = [], + restoredFiles: AttachedTextFile[] = [] + ): boolean { + // mergedIntoDraft: the restored text landed on top of a draft the user was + // already writing — both instructions now share one composer, so the caller + // must keep both their contexts rather than replacing one with the other. + const { mergedIntoDraft, droppedImages, droppedFiles } = draft.prepend({ + text, + images: restoredImages, + files: restoredFiles + }) + if (droppedImages > 0) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_IMAGES} images; ${droppedImages} restored image(s) were dropped.`, + true + ) + } + if (droppedFiles > 0) { + sendUserToast( + `You can attach up to ${MAX_ATTACHED_FILES} files; ${droppedFiles} restored file(s) were dropped.`, + true + ) + } focusInput() + return mergedIntoDraft } /** Insert a plain @filename mention for an attached file (used by the @ menu Files category). */ export function insertFileMention(name: string) { - const sep = instructions.length === 0 || instructions.endsWith(' ') ? '' : ' ' - instructions = `${instructions}${sep}${formatMention(name)} ` + const sep = draft.text.length === 0 || draft.text.endsWith(' ') ? '' : ' ' + draft.text = `${draft.text}${sep}${formatMention(name)} ` focusInput() } function clickOutside(node: HTMLElement) { function handleClick(event: MouseEvent) { + // An expanded image chip renders in a portal, so clicks in it land outside + // this node without being outside the composer. Dismissing on them would + // discard the edit the user opened the image from. + if (isImageViewerOpen()) return if (node && !node.contains(event.target as Node)) { onClickOutside() } @@ -296,33 +596,63 @@ } function sendRequest() { + // The send button is disabled while decoding, but Enter reaches here directly. + // Sending now would drop the in-flight attachments onto the following message. + if (pendingImages > 0 || pendingFiles > 0 || ingestionHolds > 0) { + return + } if (aiChatManager.loading) { // Queue the message instead of silently discarding it — it is // auto-sent when the streaming turn completes successfully. // Editing-while-loading keeps the old discard behavior. Paste // tokens are expanded into the queued text (the queue is plain - // strings), so the full content survives the auto-send. - if (editingMessageIndex === null && instructions.trim()) { - aiChatManager.queueMessage(expanded(chatDraft(instructions, pastes))) + // strings), so the full content survives the auto-send. A GLOBAL + // context-only draft counts too (mirrors the idle send guard), and + // the selection is pinned to the queued entry so the flush sends the + // chips picked at press time. + if ( + editingMessageIndex === null && + (!draft.isEmpty || (aiChatManager.mode === AIMode.GLOBAL && selectedContext.length > 0)) + ) { + const sent = draft.take() + aiChatManager.queueMessage( + expanded(chatDraft(sent.text, sent.pastes)), + sent.images, + [...selectedContext], + sent.files + ) contextTextareaComponent?.clearForSend() - instructions = '' - pastes = [] } return } if (editingMessageIndex !== null) { - aiChatManager.restartGeneration(editingMessageIndex, instructions, pastes) + // In edit mode selectedContext is the edit box's own copy (seeded from the + // message's original chips), so send exactly what's shown — the user may + // have added or removed chips. + const sent = draft.take() + aiChatManager.restartGeneration( + editingMessageIndex, + sent.text, + sent.pastes, + sent.images, + selectedContext, + sent.files + ) onEditEnd() } else { - aiChatManager.sendRequest({ instructions, pastes }) + const sent = draft.take() + aiChatManager.sendRequest({ + instructions: sent.text, + pastes: sent.pastes, + images: sent.images, + files: sent.files + }) // clearForSend() pre-zaps the textarea's mention-sync so the wipe // doesn't drop `selectedContext` before `AIChatManager.beforeSend` // snapshots it. Only mounted in SCRIPT/FLOW/GLOBAL — APP and the - // fallback textarea still rely on the plain `instructions = ''` - // reset (no `@`-mention state to coordinate). + // fallback textarea still rely on the draft reset alone (no + // `@`-mention state to coordinate). contextTextareaComponent?.clearForSend() - instructions = '' - pastes = [] } } @@ -331,7 +661,7 @@ // for the conversation bubble and expands them for the LLM inside the manager. function submitRequest() { if (onSendRequest) { - onSendRequest(expanded(chatDraft(instructions, pastes))) + onSendRequest(expanded(chatDraft(draft.text, draft.pastes))) } else { sendRequest() } @@ -503,7 +833,7 @@ } function handleAppInput(_e: Event) { - const words = instructions.split(/\s+/) + const words = draft.text.split(/\s+/) const lastWord = words[words.length - 1] if ( @@ -522,9 +852,9 @@ function handleAppContextSelection(contextElement: ContextElement) { void addContextToSelection(contextElement) // Update instructions with the selected context title - const index = instructions.lastIndexOf('@') + const index = draft.text.lastIndexOf('@') if (index !== -1) { - instructions = instructions.substring(0, index) + `@${contextElement.title}` + draft.text = draft.text.substring(0, index) + `@${contextElement.title}` } showAppContextTooltip = false } @@ -538,7 +868,20 @@ {#snippet sendStopButton()} {@const isLoading = loading ?? aiChatManager.loading} - {@const sendDisabled = disabled || instructions.trim().length === 0} + {@const emptyDraft = draft.isEmpty} + + {@const sendDisabled = + disabled || + pendingImages > 0 || + pendingFiles > 0 || + ingestionHolds > 0 || + (emptyDraft && + (onSendRequest !== undefined || + aiChatManager.mode !== AIMode.GLOBAL || + selectedContext.length === 0))} +
+ {/each} + + {#each { length: pendingImages } as _, i (i)} +
+ +
+ {/each}
{/if} {/snippet} @@ -587,22 +984,23 @@ }} > {#if isContextEnabledMode} - {#if showContext} - {@render contextPickerRow()} - {/if}
void addImages(pasted) + : undefined} + onTextFiles={aiChatManager.mode === AIMode.GLOBAL + ? (pasted) => void addTextFiles(pasted) + : undefined} {availableContext} {selectedContext} placeholder={modePlaceholder} onAddContext={(contextElement) => void addContextToSelection(contextElement)} onRemoveContext={(element) => { - selectedContext = selectedContext?.filter( - (c) => c.type !== element.type || c.title !== element.title - ) + selectedContext = selectedContext?.filter((c) => !isSameContextElement(c, element)) }} onSendRequest={() => { if (disabled) { @@ -612,7 +1010,12 @@ }} {disabled} {onKeyDown} - /> + > + {#snippet leading()} + {@render badgeRow()} + {@render imageChipsRow()} + {/snippet} + {#if !bottomRightSnippet}
{@render sendStopButton()} @@ -621,12 +1024,12 @@
{:else if aiChatManager.mode === AIMode.APP} {#if showContext} - {@render contextPickerRow()} + {@render badgeRow()} {/if}
-
{#if showContextTooltip || showCommandTooltip} diff --git a/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawers.svelte b/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawers.svelte index b73c8dc6bc..e25df2cc40 100644 --- a/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawers.svelte +++ b/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawers.svelte @@ -70,6 +70,10 @@ label: 'MQTT trigger', load: () => import('$lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte') }, + amqp: { + label: 'AMQP trigger', + load: () => import('$lib/components/triggers/amqp/AmqpTriggerEditorInner.svelte') + }, sqs: { label: 'SQS trigger', load: () => import('$lib/components/triggers/sqs/SqsTriggerEditorInner.svelte') diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index d24246b711..24d7284b95 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -3,14 +3,23 @@ import type { ChatJob, DisplayMessage } from './shared' import { expanded, messageDraft } from './chatDraft' import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' +import { scopedKey } from '$lib/userScopedStorage' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import type { PersistedContextUsage } from './tokenUsage' +import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' +import { randomUUID } from '$lib/utils/uuid' // Base IndexedDB name; userScopedDb namespaces the effective DB by the logged-in // user's email so chat messages are never physically shared across users on a // shared browser. The bare name is also the legacy (pre-namespacing) DB, claimed // once on first login. const DB_NAME = 'copilot-chat-history' +// v3 adds the images blob store (replacing v2's short-lived toolImages store). +const DB_VERSION = 3 +/** Newest image blobs kept per chat; each is a bounded (≤1568px) data URL. */ +const MAX_IMAGES_PER_CHAT = 30 +/** Marks a persisted image whose bytes live in the `images` store. */ +const IMAGE_REF_PREFIX = 'wm-image:' interface ChatSchema extends IDBSchema { chats: { @@ -37,12 +46,46 @@ interface ChatSchema extends IDBSchema { backgroundJobs?: ChatJob[] } } + // Image bytes, out-of-band from the chat record on purpose: the record is + // re-cloned into IndexedDB on every saveChat, while a blob is written once + // and read again only when its chat is reloaded. The persisted message + // arrays carry `wm-image:` refs in place of the data URLs; swapping + // happens entirely inside this class (dehydrate on save, hydrate on load), + // so live chat state never sees a ref. + images: { + key: string + value: { + id: string + chatId: string + dataUrl: string + savedAt: number + } + indexes: { 'by-chat': [string, number] } + } } function createChatStore(db: IDBPDatabase): void { if (!db.objectStoreNames.contains('chats')) { db.createObjectStore('chats', { keyPath: 'id' }) } + // v2 briefly kept full-resolution tool screenshots in their own store; the + // general blob store below covers them now. + if ((db.objectStoreNames as DOMStringList).contains('toolImages')) { + db.deleteObjectStore('toolImages' as never) + } + if (!db.objectStoreNames.contains('images')) { + const store = db.createObjectStore('images', { keyPath: 'id' }) + store.createIndex('by-chat', ['chatId', 'savedAt']) + } +} + +/** All image-blob primary keys owned by a chat (via the [chatId, savedAt] index). */ +function imageKeysForChat(db: IDBPDatabase, chatId: string) { + return db.getAllKeysFromIndex( + 'images', + 'by-chat', + IDBKeyRange.bound([chatId, -Infinity], [chatId, Infinity]) + ) } // Shared across all HistoryManager instances. Each instance owns its own @@ -97,7 +140,7 @@ export function __resetLegacyChatClaimForTesting(): void { // the `get` is O(1) on the `id` keyPath. export async function readChatModifiedItems(chatId: string): Promise { const dbh = userScopedDb(DB_NAME, { - version: 1, + version: DB_VERSION, upgrade: createChatStore, migrate: migrateLegacyChatDb }) @@ -118,7 +161,7 @@ export default class HistoryManager { // HistoryManager per AIChatManager (the singleton + one per session runtime), // so the handle must be per-instance — not a module singleton. private dbh = userScopedDb(DB_NAME, { - version: 1, + version: DB_VERSION, upgrade: createChatStore, migrate: migrateLegacyChatDb }) @@ -145,6 +188,55 @@ export default class HistoryManager { // session-tagged chats are excluded from history. private sessionId: string | undefined = $state(undefined) + // chatId+dataUrl → stable blob id, so every save of the same conversation + // maps an image to the record written the first time (write-once) instead of + // minting a new one per save. Hydration seeds it back, so a reloaded chat + // re-saves under its original ids too. Scoped by chat: each blob record has + // exactly one owning chat, so the same image pasted into two chats becomes + // two records — sharing one would let chat A's deletion or cap eviction + // destroy bytes chat B still references. + private imageIdByUrl = new Map() + + private imageIdKey(chatId: string, dataUrl: string): string { + return chatId + '\n' + dataUrl + } + + // Blob writes, stale-blob deletes, and the record put span several IndexedDB + // transactions, and saveChat has concurrent callers (turn saves, the + // modified-items and background-jobs writers). Interleaved, an older save's + // delete pass can remove a blob a newer save just verified, landing the + // newer record with a dangling ref — so every DB write runs through this + // per-manager queue. A failed write is rethrown to its caller without + // wedging the queue. + private dbWriteQueue: Promise = Promise.resolve() + + private enqueueDbWrite(op: (db: IDBPDatabase) => Promise): Promise { + // A write belongs to the user who initiated it: capture the scoped DB name + // now and skip execution if the logged-in user changed while queued — + // resolving the handle only at execution time would write this user's chat + // into the NEXT user's database on an in-place account switch. + const name = scopedKey(DB_NAME) + const exec = async () => { + if (!name || scopedKey(DB_NAME) !== name) return + const db = await this.dbh.whenReady() + if (!db || db.name !== name) return + return op(db) + } + const run = this.dbWriteQueue.then(exec, exec) + this.dbWriteQueue = run.catch(() => {}) + return run + } + + /** Drop cached blob ids of every chat but the given one, so the map doesn't + * pin past chats' data URL strings in memory for the whole session (a + * reopened chat re-seeds its ids through hydration). */ + private pruneImageIds(keepChatId: string) { + const prefix = keepChatId + '\n' + for (const key of this.imageIdByUrl.keys()) { + if (!key.startsWith(prefix)) this.imageIdByUrl.delete(key) + } + } + private pastChats = $derived( Object.values(this.savedChats) .filter((c) => c.id !== this.currentChatId) @@ -153,6 +245,9 @@ export default class HistoryManager { ) async init() { + // (Re)initializing adopts a new identity's history: drop the previous + // identity's cached blob ids with it. + this.imageIdByUrl.clear() // whenReady() is email-gated (returns undefined before the user is known — // all callers run post-login, and the singleton re-inits via onUserChange), // runs the legacy migration once, and reopens automatically on user change. @@ -194,10 +289,7 @@ export default class HistoryManager { const snapshot = $state.snapshot(existing) const updated = { ...snapshot, sessionId } this.savedChats = { ...this.savedChats, [chatId]: updated } - // Resolve the DB via the handle (not a cached ref) so a write always lands - // in the current user's DB, even after an in-place user switch. - const db = await this.dbh.whenReady() - if (db) await db.put('chats', updated) + await this.enqueueDbWrite((db) => db.put('chats', updated)) } getPastChats() { @@ -216,6 +308,168 @@ export default class HistoryManager { return this.savedChats[id]?.backgroundJobs } + /** + * Swap every inline image (data URL) in the given message arrays for a + * `wm-image:` ref, mutating them in place — callers pass a clone bound + * for IndexedDB, never live chat state or the in-memory savedChats mirror. + * Returns the id → dataUrl map of every image the arrays reference, plus + * every reference (pre-existing refs included) in walk order — + * persistImageBlobs ranks ids by their newest reference, so the transcript + * walks FIRST: it is always whole and chronological, while drop-oldest + * compaction removes old API messages, which would misorder a dropped + * message's still-displayed image. + */ + private dehydrateImages( + chatId: string, + actualMessages: ChatCompletionMessageParam[], + displayMessages: DisplayMessage[] + ): { blobs: Map; refs: string[] } { + const blobs = new Map() + const refs: string[] = [] + const refFor = (url: string): string => { + // Already a ref (a record that was never rehydrated): it still counts + // as a reference — omitting it would let the stale-delete pass reclaim + // its blob — but there are no bytes to (re)write. + if (url.startsWith(IMAGE_REF_PREFIX)) { + refs.push(url.slice(IMAGE_REF_PREFIX.length)) + return url + } + const key = this.imageIdKey(chatId, url) + let id = this.imageIdByUrl.get(key) + if (!id) { + id = randomUUID() + this.imageIdByUrl.set(key, id) + } + blobs.set(id, url) + refs.push(id) + return IMAGE_REF_PREFIX + id + } + for (const message of displayMessages) { + if (message.role === 'user' && message.images) { + for (const image of message.images) { + if (image.dataUrl.startsWith('data:') || image.dataUrl.startsWith(IMAGE_REF_PREFIX)) { + image.dataUrl = refFor(image.dataUrl) + } + } + } else if ( + message.role === 'tool' && + (message.imageUrl?.startsWith('data:') || message.imageUrl?.startsWith(IMAGE_REF_PREFIX)) + ) { + message.imageUrl = refFor(message.imageUrl) + } + } + for (const message of actualMessages) { + if (!Array.isArray(message.content)) continue + for (const part of message.content as any[]) { + if ( + part?.type === 'image_url' && + (part.image_url?.url?.startsWith('data:') || + part.image_url?.url?.startsWith(IMAGE_REF_PREFIX)) + ) { + part.image_url.url = refFor(part.image_url.url) + } + } + } + return { blobs, refs } + } + + /** + * The record's newest MAX_IMAGES_PER_CHAT distinct images, ranked by their + * LAST reference — the exact set of blobs the chat should own once the + * record is committed. The saved record is the single source of truth: + * deriving the set from it (rather than from persisted write times) keeps + * eviction deterministic and idempotent when turns are truncated, identical + * bytes are re-attached, or compaction rewrites the arrays. A ref outside + * the kept set hydrates to the omitted-image placeholder. + */ + private keptImageIds(refs: string[]): Set { + const keep = new Set() + for (let i = refs.length - 1; i >= 0 && keep.size < MAX_IMAGES_PER_CHAT; i--) { + keep.add(refs[i]) + } + return keep + } + + private async writeKeptImageBlobs( + db: IDBPDatabase, + chatId: string, + blobs: Map, + keep: Set + ) { + for (const id of keep) { + const dataUrl = blobs.get(id) + if (dataUrl !== undefined && (await db.getKey('images', id)) === undefined) { + await db.put('images', { id, chatId, dataUrl, savedAt: Date.now() }) + } + } + } + + private async deleteStaleImageBlobs( + db: IDBPDatabase, + chatId: string, + keep: Set + ) { + for (const key of await imageKeysForChat(db, chatId)) { + if (!keep.has(key)) await db.delete('images', key) + } + } + + /** + * Resolve every `wm-image:` ref in the chat clone back to its data URL, + * in place. A missing blob (evicted by the per-chat cap, or IndexedDB + * unavailable altogether) degrades the API part to the omitted-image + * placeholder and drops the transcript copy — a ref must never leak into + * bubbles or outgoing requests. Inline data URLs (records persisted before + * the blob store) pass through untouched. + */ + private async hydrateImages( + db: IDBPDatabase | undefined, + chatId: string, + actualMessages: ChatCompletionMessageParam[], + displayMessages: DisplayMessage[] + ) { + const load = async (ref: string): Promise => { + const id = ref.slice(IMAGE_REF_PREFIX.length) + const dataUrl = (await db?.get('images', id))?.dataUrl + if (dataUrl) this.imageIdByUrl.set(this.imageIdKey(chatId, dataUrl), id) + return dataUrl + } + for (const message of actualMessages) { + if (!Array.isArray(message.content)) continue + const content = message.content as any[] + for (let i = 0; i < content.length; i++) { + const part = content[i] + if (part?.type === 'image_url' && part.image_url?.url?.startsWith(IMAGE_REF_PREFIX)) { + const dataUrl = await load(part.image_url.url) + content[i] = dataUrl + ? { ...part, image_url: { ...part.image_url, url: dataUrl } } + : { type: 'text', text: IMAGE_OMITTED_PLACEHOLDER } + } + } + } + for (const message of displayMessages) { + if (message.role === 'user' && message.images) { + const images: AttachedImage[] = [] + for (const image of message.images) { + if (!image.dataUrl.startsWith(IMAGE_REF_PREFIX)) { + images.push(image) + continue + } + const dataUrl = await load(image.dataUrl) + if (dataUrl) images.push({ ...image, dataUrl }) + } + message.images = images.length > 0 ? images : undefined + // An image-only bubble that lost every image would render empty — + // say what happened instead. + if (!message.images && !message.content.trim()) { + message.content = IMAGE_OMITTED_PLACEHOLDER + } + } else if (message.role === 'tool' && message.imageUrl?.startsWith(IMAGE_REF_PREFIX)) { + message.imageUrl = await load(message.imageUrl) + } + } + } + async saveChat( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], @@ -231,12 +485,26 @@ export default class HistoryManager { // expanding collapsed-paste tokens so it reads as text rather than the // chip label + its zero-width id chars. const existingTitle = this.savedChats[this.currentChatId]?.title + const titleSource = displayMessages.find((m) => m.role !== 'summary') ?? displayMessages[0] + const derivedTitle = expanded(messageDraft(titleSource)).slice(0, 50) + // An attachment-only first turn has no text to derive from — fall back to + // the attachment's filename so the History menu entry isn't blank. + const imageFallback = + titleSource.role === 'user' + ? titleSource.images?.length + ? (titleSource.images[0].name ?? 'Image attachment') + : titleSource.files?.length + ? titleSource.files[0].name + : '' + : '' + // A hydrated omission marker is not user text — deriving from it would + // overwrite the filename title an evicted image-only chat was given. const title = displayMessages[0].role === 'summary' && existingTitle !== undefined ? existingTitle - : expanded( - messageDraft(displayMessages.find((m) => m.role !== 'summary') ?? displayMessages[0]) - ).slice(0, 50) + : derivedTitle.trim() && derivedTitle !== IMAGE_OMITTED_PLACEHOLDER + ? derivedTitle + : imageFallback || existingTitle || '' // we don't want to save the snapshot in the history const updatedChat = { actualMessages: $state.snapshot(messages), @@ -273,13 +541,38 @@ export default class HistoryManager { } : {}) } + // The mirror mirrors what the DB holds (refs — the snapshot is + // dehydrated below before either sees it): a reopened chat hydrates + // through the store, reseeding stable blob ids. When IndexedDB is + // unavailable the writes no-op and hydration degrades the refs to + // omitted-image placeholders — like every other userScopedDb consumer, + // history simply doesn't persist there. + const { blobs, refs } = this.dehydrateImages( + updatedChat.id, + updatedChat.actualMessages, + updatedChat.displayMessages + ) this.savedChats = { ...this.savedChats, [updatedChat.id]: updatedChat } - - const db = await this.dbh.whenReady() - if (db) await db.put('chats', updatedChat) + await this.enqueueDbWrite(async (db) => { + // Write order is the crash-safety story: kept blobs land before the + // record that references them, and stale blobs are deleted only after + // the new record is committed. A failure at any step leaves the last + // committed record fully hydratable — at worst orphan blobs linger + // until the next successful save's delete pass reclaims them. + const keep = this.keptImageIds(refs) + await this.writeKeptImageBlobs(db, updatedChat.id, blobs, keep) + await db.put('chats', updatedChat) + // Best-effort: the record is already committed, so a failed cleanup + // (e.g. a user switch closed this handle mid-op) must not turn a + // successful save into a rejection — the orphans are reclaimed by + // the next successful save's pass. + await this.deleteStaleImageBlobs(db, updatedChat.id, keep).catch((err) => + console.error('Could not prune stale image blobs', err) + ) + }) } } @@ -292,20 +585,32 @@ export default class HistoryManager { ) { await this.saveChat(displayMessages, messages, contextUsage, modifiedItems, backgroundJobs) this.currentChatId = createLongHash() + this.pruneImageIds(this.currentChatId) } deletePastChat(id: string) { this.savedChats = Object.fromEntries( Object.entries(this.savedChats).filter(([key]) => key !== id) ) - void this.dbh.whenReady().then((db) => db?.delete('chats', id)) + void this.enqueueDbWrite(async (db) => { + await db.delete('chats', id) + const keys = await imageKeysForChat(db, id) + await Promise.all(keys.map((key) => db.delete('images', key))) + }).catch((err) => console.error('Could not delete chat', err)) } - loadPastChat(id: string) { + async loadPastChat(id: string) { const chat = this.savedChats[id] - if (chat) { - this.currentChatId = id - return chat - } + if (!chat) return + this.currentChatId = id + this.pruneImageIds(id) + // Hand back a hydrated clone: the stored record keeps its refs (matching + // what the DB holds) while the live chat gets real data URLs. Hydration + // runs even without a DB so refs degrade to placeholders instead of + // leaking into bubbles and requests. + const snapshot = $state.snapshot(chat) as typeof chat + const db = await this.dbh.whenReady() + await this.hydrateImages(db, id, snapshot.actualMessages, snapshot.displayMessages) + return snapshot } } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 540dfaf72e..ffcc065157 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -90,6 +90,449 @@ describe('HistoryManager legacy chat-history migration', () => { expect(hm.getAllSavedChats()).toEqual([]) }) + it('persists image bytes out of the chat record and hydrates them back on load', async () => { + const png = 'data:image/png;base64,FULLBYTES' + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { role: 'user', content: 'look', images: [{ dataUrl: png, mediaType: 'image/png' }] }, + { role: 'tool', tool_call_id: 't1', content: 'shot', imageUrl: png } + ] as DisplayMessage[], + [ + { + role: 'user', + content: [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: png } } + ] + } + ] as ChatCompletionMessageParam[] + ) + + // The chat record holds refs, not bytes — and the shared data URL of the + // bubble, tool card, and API part dedups to a single blob record. + const db = await openDB('copilot-chat-history::admin@test') + const record = await db.get('chats' as never, chatId) + expect(JSON.stringify(record)).not.toContain('FULLBYTES') + expect((record as any).actualMessages[0].content[1].image_url.url).toMatch(/^wm-image:/) + expect(await db.count('images' as never)).toBe(1) + db.close() + + // A fresh instance (reload) hydrates the refs back to the original bytes. + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[1].image_url.url).toBe(png) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + expect((chat?.displayMessages[1] as any).imageUrl).toBe(png) + }) + + it('re-saving the same conversation does not mint new blob records', async () => { + const png = 'data:image/png;base64,STABLE' + const display = [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[] + const hm = new HistoryManager() + await hm.init() + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(1) + db.close() + }) + + it('caps stored blobs per chat; an evicted ref hydrates to the omitted placeholder', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const messages = [] as ChatCompletionMessageParam[] + for (let i = 0; i <= 30; i++) { + messages.push({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(i) } }] + } as ChatCompletionMessageParam) + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + } + // Re-saving the over-cap chat must not resurrect the evicted oldest blob + // (its live data URL is still in the arrays): a re-put would stamp it + // newest and push the eviction onto a newer image, and repeated saves + // would rotate the hole toward the latest attachment. + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + await hm.saveChat([{ role: 'user', content: 'x' }] as DisplayMessage[], messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect((chat?.actualMessages[1].content as any[])[0].image_url.url).toBe(urlFor(1)) + expect((chat?.actualMessages[30].content as any[])[0].image_url.url).toBe(urlFor(30)) + }) + + it('keeps blob chronology when drop-oldest compaction removed old API counterparts', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + // Transcript keeps all 31 bubbles; drop-oldest compaction pruned the API + // history down to the newest 4 image messages. + const display = Array.from({ length: 31 }, (_, i) => ({ + role: 'user', + content: 'x', + index: i - 27, + images: [{ dataUrl: urlFor(i), mediaType: 'image/png' }] + })) as DisplayMessage[] + const messages = Array.from({ length: 4 }, (_, i) => ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(27 + i) } }] + })) as ChatCompletionMessageParam[] + await hm.saveChat(display, messages) + await hm.saveChat(display, messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + // The oldest transcript image is the one over the cap... + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + // ...never a newer one that merely lost its API counterpart ordering. + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(urlFor(27)) + expect((chat?.displayMessages[30] as any).images[0].dataUrl).toBe(urlFor(30)) + }) + + it('truncated turns release their blobs from the cap', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const imageMsg = (i: number) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url: urlFor(i) } }] + }) as ChatCompletionMessageParam + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + // Fill the cap exactly, then retry/edit truncates the tail to 5 messages + // and adds one replacement image. The truncated turns' blobs must stop + // counting against the cap — image 0 is among the newest 6 *referenced* + // images and must survive. + await hm.saveChat( + display, + Array.from({ length: 30 }, (_, i) => imageMsg(i)) + ) + await hm.saveChat(display, [...Array.from({ length: 5 }, (_, i) => imageMsg(i)), imageMsg(99)]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(6) + db.close() + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(urlFor(0)) + expect((chat?.actualMessages[5].content as any[])[0].image_url.url).toBe(urlFor(99)) + }) + + it('re-attaching identical bytes ranks the image by its newest reference', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + const imageMsg = (url: string) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url } }] + }) as ChatCompletionMessageParam + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + const reused = 'data:image/png;base64,REUSED' + // The reused image appears first, 30 distinct images follow, then it is + // attached again. Its newest reference makes it one of the newest 30 + // distinct images, so the eviction must land on the oldest of the middle + // ones — not on the image the user just re-attached. + const messages = [ + imageMsg(reused), + ...Array.from({ length: 30 }, (_, i) => imageMsg(urlFor(i))), + imageMsg(reused) + ] + await hm.saveChat(display, [messages[0]]) + await hm.saveChat(display, messages) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(reused) + expect((chat?.actualMessages[31].content as any[])[0].image_url.url).toBe(reused) + expect((chat?.actualMessages[1].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect((chat?.actualMessages[2].content as any[])[0].image_url.url).toBe(urlFor(1)) + }) + + it('a save overlapping an older save keeps every blob its record references', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const X = 'data:image/png;base64,XBYTES' + const Y = 'data:image/png;base64,YBYTES' + const display = [{ role: 'user', content: 'x' }] as DisplayMessage[] + const msg = (url: string) => + ({ + role: 'user', + content: [{ type: 'image_url', image_url: { url } }] + }) as ChatCompletionMessageParam + await hm.saveChat(display, [msg(X)]) + + // An overlapping pair: the older snapshot no longer references X (retry + // truncation), the newer one re-references it (as its newest image) and + // adds Y. Un-serialized, the older save's delete pass removes X's blob + // after the newer save verified its existence and moved on, landing the + // winning record with a dangling ref. + const older = hm.saveChat(display, [ + { role: 'user', content: 'no images' } as ChatCompletionMessageParam + ]) + const newer = hm.saveChat(display, [msg(Y), msg(X)]) + await Promise.all([older, newer]) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(Y) + expect((chat?.actualMessages[1].content as any[])[0].image_url.url).toBe(X) + }) + + it('reopening a rotated chat reuses its blob records on the next save', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const png = 'data:image/png;base64,STABLEBYTES' + await hm.save( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) // rotates to a fresh chat, pruning the id cache + + const before = await openDB('copilot-chat-history::admin@test') + const idsBefore = await before.getAllKeys('images' as never) + before.close() + + // Reopening must reseed the stable blob id — a re-save that minted a new + // id would rewrite every blob (and delete the old ones) on each reopen. + const chat = await hm.loadPastChat(chatId) + await hm.saveChat(chat!.displayMessages as DisplayMessage[], chat!.actualMessages) + + const after = await openDB('copilot-chat-history::admin@test') + const idsAfter = await after.getAllKeys('images' as never) + after.close() + expect(idsAfter).toEqual(idsBefore) + }) + + it('the same image in two chats gets two owned blobs; deleting one chat spares the other', async () => { + const png = 'data:image/png;base64,SHAREDBYTES' + const display = [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[] + const hm = new HistoryManager() + await hm.init() + const chatA = hm.getCurrentChatId() + await hm.save(display, [] as ChatCompletionMessageParam[]) // rotates to a new chat + const chatB = hm.getCurrentChatId() + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const db = await openDB('copilot-chat-history::admin@test') + expect(await db.count('images' as never)).toBe(2) + db.close() + + hm.deletePastChat(chatA) + await vi.waitFor(async () => { + const d = await openDB('copilot-chat-history::admin@test') + const count = await d.count('images' as never) + d.close() + expect(count).toBe(1) + }) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatB) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it("deletes a chat's image blobs along with the chat", async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: 'x', + images: [{ dataUrl: 'data:image/png;base64,GONE', mediaType: 'image/png' }] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + hm.deletePastChat(chatId) + + await vi.waitFor(async () => { + const db = await openDB('copilot-chat-history::admin@test') + const count = await db.count('images' as never) + db.close() + expect(count).toBe(0) + }) + }) + + it('loads pre-blob-store records with inline data URLs untouched', async () => { + const png = 'data:image/png;base64,LEGACYINLINE' + const hm = new HistoryManager() + await hm.init() + // Simulate a record persisted before the blob store existed. + const db = await openDB('copilot-chat-history::admin@test') + await db.put( + 'chats' as never, + { + id: 'legacy1', + title: 'legacy', + lastModified: 1, + actualMessages: [ + { role: 'user', content: [{ type: 'image_url', image_url: { url: png } }] } + ], + displayMessages: [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] + } as never + ) + db.close() + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat('legacy1') + expect((chat?.actualMessages[0].content as any[])[0].image_url.url).toBe(png) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it("a failed record put cannot orphan the previous record's blobs", async () => { + const png = 'data:image/png;base64,SURVIVES' + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + await hm.saveChat( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + + // Make the next `chats` put fail (quota/connection failure), on a save + // whose record drops the image — its blob is now stale, but deleting it + // before the record commit would corrupt the still-current OLD record. + const probe = await openDB('probe-proto', 1, { + upgrade: (d) => { + d.createObjectStore('s') + } + }) + const proto = Object.getPrototypeOf( + probe.transaction('s' as never, 'readwrite').objectStore('s' as never) + ) + probe.close() + const origPut = proto.put + let failNext = true + proto.put = function (this: { name: string }, ...args: unknown[]) { + if (this.name === 'chats' && failNext) { + failNext = false + throw new Error('simulated quota failure') + } + return origPut.apply(this, args) + } + try { + await expect( + hm.saveChat( + [{ role: 'user', content: 'no image' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + ).rejects.toThrow('simulated quota failure') + } finally { + proto.put = origPut + } + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images[0].dataUrl).toBe(png) + }) + + it('drops queued writes when the user switches before they execute (no cross-user leak)', async () => { + const hm = new HistoryManager() + await hm.init() + const display = [ + { + role: 'user', + content: 'private to A', + images: [{ dataUrl: 'data:image/png;base64,LEAKBYTES', mediaType: 'image/png' }] + } + ] as DisplayMessage[] + // Enqueue two writes under user A, then switch identity before either + // executes. Resolving the DB handle at execution time would write A's + // chat and image blob into B's database. + const first = hm.saveChat(display, [] as ChatCompletionMessageParam[]) + const second = hm.saveChat(display, [] as ChatCompletionMessageParam[]) + userStore.set(asUser('other@test')) + await Promise.all([first, second]) + + const db = await openDB('copilot-chat-history::other@test') + const chats = db.objectStoreNames.contains('chats') ? await db.count('chats' as never) : 0 + const images = db.objectStoreNames.contains('images') ? await db.count('images' as never) : 0 + db.close() + expect(chats).toBe(0) + expect(images).toBe(0) + }) + + it("a save finishing after another user's init cannot leak into their mirror", async () => { + const hm = new HistoryManager() + await hm.init() + + // The save passes enqueueDbWrite's identity checks under user A; the + // account switches (and re-inits, as the platform does on user change) + // while its transaction is still running. The committed save must still + // resolve (the switch closes A's handle, failing only the best-effort + // cleanup tail) and its convergence must not merge A's record into B's + // freshly adopted mirror. + const probe = await openDB('probe-proto2', 1, { + upgrade: (d) => { + d.createObjectStore('s') + } + }) + const proto = Object.getPrototypeOf( + probe.transaction('s' as never, 'readwrite').objectStore('s' as never) + ) + probe.close() + const origPut = proto.put + let initDone: Promise | undefined + proto.put = function (this: { name: string }, ...args: unknown[]) { + if (this.name === 'chats' && !initDone) { + userStore.set(asUser('other@test')) + initDone = hm.init() + } + return origPut.apply(this, args) + } + try { + await hm.saveChat( + [{ role: 'user', content: 'private to A' }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + } finally { + proto.put = origPut + } + await initDone + + expect(hm.getAllSavedChats()).toEqual([]) + }) + it('writes land in the current user DB after an in-place user switch', async () => { const hm = new HistoryManager() await hm.init() @@ -115,6 +558,119 @@ describe('HistoryManager legacy chat-history migration', () => { }) }) +describe('HistoryManager image-only chats', () => { + it('titles an image-only chat from its attachment instead of leaving it blank', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: '', + images: [ + { dataUrl: 'data:image/png;base64,A', mediaType: 'image/png', name: 'mockup.png' } + ] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('mockup.png') + }) + + it('keeps the filename title when the evicted bubble re-saves as an omission marker', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + await hm.saveChat( + [ + { + role: 'user', + content: '', + images: [ + { dataUrl: 'data:image/png;base64,A', mediaType: 'image/png', name: 'mockup.png' } + ] + } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + // Post-reload shape of an evicted image-only first bubble: the omission + // marker as content, images gone. Re-saving must not adopt the marker as + // the chat's title. + await hm.saveChat( + [ + { role: 'user', content: '[image omitted]' }, + { role: 'user', content: 'follow-up' } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('mockup.png') + }) + + it('shows an omission marker when an evicted image-only bubble reloads', async () => { + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const urlFor = (i: number) => `data:image/png;base64,IMG${String(i).padStart(2, '0')}` + // 31 image-only turns: the oldest exceeds the blob cap, so its bubble + // reloads with no image AND no text — it must say what happened instead + // of rendering empty. + const display = Array.from({ length: 31 }, (_, i) => ({ + role: 'user', + content: '', + index: i, + images: [{ dataUrl: urlFor(i), mediaType: 'image/png' }] + })) as DisplayMessage[] + await hm.saveChat(display, [] as ChatCompletionMessageParam[]) + + const reloaded = new HistoryManager() + await reloaded.init() + const chat = await reloaded.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + expect((chat?.displayMessages[0] as any).content).toBe('[image omitted]') + expect((chat?.displayMessages[1] as any).images[0].dataUrl).toBe(urlFor(1)) + expect((chat?.displayMessages[1] as any).content).toBe('') + }) +}) + +describe('HistoryManager without IndexedDB', () => { + it('degrades cleanly: refs never leak into bubbles or message content', async () => { + // whenReady() resolves undefined when opens fail (private browsing, + // blocked, corrupt). History then simply doesn't persist — like every + // other userScopedDb consumer — but a reloaded chat must degrade its + // unresolvable refs to omitted-image placeholders, never hand raw + // `wm-image:` URLs to an or an outgoing request. + ;(globalThis as any).indexedDB = { + open: () => { + throw new Error('blocked') + } + } + const hm = new HistoryManager() + await hm.init() + const chatId = hm.getCurrentChatId() + const png = 'data:image/png;base64,MEMORYONLY' + await hm.saveChat( + [ + { role: 'user', content: 'x', images: [{ dataUrl: png, mediaType: 'image/png' }] } + ] as DisplayMessage[], + [ + { + role: 'user', + content: [{ type: 'image_url', image_url: { url: png } }] + } + ] as ChatCompletionMessageParam[] + ) + + const chat = await hm.loadPastChat(chatId) + expect((chat?.displayMessages[0] as any).images).toBeUndefined() + expect((chat?.actualMessages[0].content as any[])[0]).toEqual({ + type: 'text', + text: '[image omitted]' + }) + expect(JSON.stringify(chat)).not.toContain('wm-image:') + }) +}) + describe('HistoryManager title across compaction', () => { it('keeps the original title once a summary boundary leads the transcript', async () => { const hm = new HistoryManager() @@ -142,6 +698,39 @@ describe('HistoryManager title across compaction', () => { }) }) +describe('HistoryManager mirror convergence under concurrent metadata saves', () => { + it('an older save completing mid-stream cannot erase newer metadata', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + const display = [{ role: 'user', content: 'x', index: 0 }] as DisplayMessage[] + const job = { id: 'job1', status: 'running' } as unknown as ChatJob + + // s1 carries modifiedItems; s2 (overlapping) carries backgroundJobs and + // inherits s1's modifiedItems from the mirror. Awaiting s1 lets its + // convergence run while s2 is still queued — it must not rewind the + // mirror, or s3's backgroundJobs fallback below reads the stale record + // and permanently erases the job. + const p1 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, ['script:a']) + const p2 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, undefined, [job]) + await p1 + const p3 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, [ + 'script:a', + 'script:b' + ]) + await Promise.all([p2, p3]) + + expect(hm.getModifiedItems(id)).toEqual(['script:a', 'script:b']) + expect(hm.getBackgroundJobs(id)).toEqual([job]) + + const db = await openDB('copilot-chat-history::admin@test') + const record = (await db.get('chats' as never, id)) as any + db.close() + expect(record.modifiedItems).toEqual(['script:a', 'script:b']) + expect(record.backgroundJobs).toEqual([job]) + }) +}) + describe('HistoryManager modified-items mask persistence', () => { const msgs = [{ role: 'user', content: 'hello', index: 0 }] as DisplayMessage[] diff --git a/frontend/src/lib/components/copilot/chat/JobsSegment.svelte b/frontend/src/lib/components/copilot/chat/JobsSegment.svelte index 0bcb315bda..b8a59d5bc9 100644 --- a/frontend/src/lib/components/copilot/chat/JobsSegment.svelte +++ b/frontend/src/lib/components/copilot/chat/JobsSegment.svelte @@ -3,11 +3,11 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import Portal from '$lib/components/Portal.svelte' - import Popover from '$lib/components/meltComponents/Popover.svelte' + import SessionStatusPopover from '$lib/components/sessions/SessionStatusPopover.svelte' import { zIndexes } from '$lib/zIndexes' import JobStatusIcon from '$lib/components/runs/JobStatusIcon.svelte' import FlowStatusWaitingForEvents from '$lib/components/FlowStatusWaitingForEvents.svelte' - import { ChevronUp, ExternalLink, Hourglass, ThumbsUp, TimerOff } from 'lucide-svelte' + import { ChevronUp, Hourglass, ThumbsUp, TimerOff } from 'lucide-svelte' import { base } from '$lib/base' import { slide } from 'svelte/transition' import { JobService, type Job } from '$lib/gen' @@ -15,6 +15,7 @@ import { sendUserToast } from '$lib/toast' import { getAiChatManager } from './aiChatManagerContext' import { deriveChatJobStatus, type ChatJob, type ChatJobStatus } from './shared' + import { TOKEN_TRIGGER_CLASS } from '$lib/components/sessions/SessionStatusToken.svelte' // The "Jobs" segment of the session bar: a compact status chip that summarizes // the background jobs the chat started, opening a popover with the full list @@ -31,8 +32,21 @@ const queuedCount = $derived( jobs.filter((j) => j.status === 'queued' || j.status === 'scheduled').length ) - const failureCount = $derived(jobs.filter((j) => j.status === 'failure').length) - const successCount = $derived(jobs.filter((j) => j.status === 'success').length) + // A finished job counts as reviewed once its terminal status has been shown in + // the open popover; a job that finishes while the popover is closed starts + // unreviewed. The flag lives on the job (persisted with it), so review state + // survives a reload. + $effect(() => { + if (!open) return + aiChatManager.markJobsReviewed(jobs.filter((j) => isTerminal(j.status)).map((j) => j.jobId)) + }) + // Only unreviewed jobs feed the terminal readout: an outcome the user already + // saw must not resurface on the chip when a later job finishes. + const unreviewed = $derived(jobs.filter((j) => !j.reviewed)) + const allReviewed = $derived(jobs.length > 0 && unreviewed.length === 0) + + const failureCount = $derived(unreviewed.filter((j) => j.status === 'failure').length) + const successCount = $derived(unreviewed.filter((j) => j.status === 'success').length) const liveCount = $derived(jobs.filter((j) => !isTerminal(j.status)).length) const hasLive = $derived(liveCount > 0) @@ -69,9 +83,18 @@ // Aggregate chip readout, priority-ordered so the most action-worthy state // wins the dot: approval > running > queued > failed > succeeded. A live run - // takes the dot even if an earlier job failed (failure resurfaces once idle). + // takes the dot even if an earlier job failed (an unreviewed failure + // resurfaces once idle). Once every finished job has been reviewed in the + // popover, the chip relaxes to a neutral executed-count. const segment = $derived.by( (): { dot: string; pulse: boolean; text: string; danger: boolean } => { + if (allReviewed) + return { + dot: 'bg-gray-400', + pulse: false, + text: `${jobs.length} job${jobs.length === 1 ? '' : 's'} executed`, + danger: false + } if (approvalCount > 0) return { dot: dotClass('suspended'), @@ -104,12 +127,27 @@ danger: false } if (failureCount > 0) - return { dot: dotClass('failure'), pulse: false, text: `${jobs.length}`, danger: true } - // All terminal, none failed: green if anything actually succeeded, else gray - // (only canceled jobs left — a cancel isn't a success, so don't show green). + return { + dot: dotClass('failure'), + pulse: false, + text: `${failureCount} failed`, + danger: true + } + // All terminal, nothing unreviewed failed: green if anything unreviewed + // succeeded, else gray (only canceled left — a cancel isn't a success). if (successCount > 0) - return { dot: dotClass('success'), pulse: false, text: `${jobs.length}`, danger: false } - return { dot: dotClass('canceled'), pulse: false, text: `${jobs.length}`, danger: false } + return { + dot: dotClass('success'), + pulse: false, + text: `${successCount} succeeded`, + danger: false + } + return { + dot: dotClass('canceled'), + pulse: false, + text: `${unreviewed.length} canceled`, + danger: false + } } ) @@ -191,7 +229,6 @@ } // --- Popover open state + auto-open on approval --- - let popover: Popover | undefined = $state() let open = $state(false) // A job entering the approval state needs attention, so open the popover to @@ -200,7 +237,7 @@ let prevApprovalCount = 0 $effect(() => { const count = approvalCount - if (count > prevApprovalCount) popover?.open() + if (count > prevApprovalCount) open = true prevApprovalCount = count }) @@ -256,26 +293,33 @@ even when the chip's visual change alone wouldn't be. role="status" already implies aria-live="polite". -->
{announcement}
- job.jobId} + rowTitle={(job) => job.label} + onPick={openRun} + placement={standalone ? 'top-end' : 'top-start'} usePointerDownOutside - class={standalone + closeOnOtherPopoverOpen={!standalone} + triggerClass={standalone ? 'flex h-[34px] w-full items-center rounded-md border bg-surface-tertiary px-3 hover:bg-surface-hover' - : 'flex h-full items-center px-3.5 hover:bg-surface-hover'} - triggerAttrs={{ 'aria-label': ariaLabel, 'aria-haspopup': 'dialog' }} - contentClasses="!bg-surface" + : TOKEN_TRIGGER_CLASS} + maxHeightClass={standalone ? 'max-h-[50vh]' : 'max-h-[min(12rem,50vh)]'} > - {#snippet trigger()} + {#snippet customTrigger()} - Jobs + {#if standalone} + Jobs + {/if} @@ -294,62 +338,42 @@ {/if} {/snippet} - {#snippet content()} -
-
Jobs this session
-
- {#each sortedJobs as job (job.jobId)} -
- {#if job.status === 'queued' || !job.job} - - - {:else} - - {/if} - {job.label} - {elapsedLabel(job)} -
- {#if job.status === 'suspended'} - - {/if} - {#if !isTerminal(job.status)} - - {/if} -
-
- {/each} -
-
+ {#snippet row(job)} + {#if job.status === 'queued' || !job.job} + + + {:else} + + {/if} + {job.label} + {elapsedLabel(job)} {/snippet} -
+ {#snippet actions(job)} + {#if job.status === 'suspended'} + + {/if} + {#if !isTerminal(job.status)} + + {/if} + {/snippet} + +{#if aiChatManager.queuedMessage || aiChatManager.queuedImages.length > 0 || aiChatManager.queuedFiles.length > 0 || (aiChatManager.queuedContext?.length ?? 0) > 0}
-

- {aiChatManager.queuedMessage} -

+ {#if aiChatManager.queuedImages.length > 0} +
+ {#each aiChatManager.queuedImages as image, i (i)} + {image.name + {/each} +
+ {/if} + {#if aiChatManager.queuedFiles.length > 0} +
+ {#each aiChatManager.queuedFiles as file, i (i)} + + + {file.name} + + {/each} +
+ {/if} + {#if aiChatManager.queuedMessage} +

+ {aiChatManager.queuedMessage} +

+ {:else if aiChatManager.queuedImages.length === 0 && aiChatManager.queuedFiles.length === 0 && aiChatManager.queuedContext?.length} +
+ {#each aiChatManager.queuedContext as element (contextElementKey(element))} + + {/each} +
+ {/if}
+ {/snippet} + + + {#if showPreviewChip && message.previewCard} +
+ {@render headerButton()} +
- + {:else} + {@render headerButton()} + {/if} + + + {#if message.imageUrl} +
+ +
+ {/if} {#if isExpanded} @@ -137,6 +179,8 @@ {#if visibleActions.length > 0} + {:else if message.webSearchSources?.length && !message.error} + {:else} + import { PanelRight } from 'lucide-svelte' + import { Button } from '$lib/components/common' + import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import type { IconType } from '$lib/utils' + import { runToolDisplayAction } from './createdResourceActions.svelte' + import { openItemPreviewAction, type PreviewCardKind } from './shared' + + interface Props { + card: { kind: PreviewCardKind; path: string } + } + + let { card }: Props = $props() + + const kindLabel = $derived(card.kind === 'raw_app' ? 'app' : card.kind) + + let opening = $state(false) + async function open() { + if (opening) return + opening = true + try { + await runToolDisplayAction(openItemPreviewAction(card.kind, card.path)) + } finally { + opening = false + } + } + + + diff --git a/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte b/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte new file mode 100644 index 0000000000..ff71c44160 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/WebSearchSourcesDisplay.svelte @@ -0,0 +1,83 @@ + + +
+ Sources: + +
diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts index 9c9a4d0bb0..1ac413973f 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.test.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' -import { convertOpenAIToAnthropicMessages } from './anthropic' +import { convertOpenAIToAnthropicMessages, partialWebSearchQuery } from './anthropic' // anthropic.ts pulls in the chat client/registry layer at import time; the // converter under test is pure, so stub those side-effecting modules away. @@ -14,7 +14,8 @@ vi.mock('../reasoningRegistry', () => ({ })) vi.mock('./shared', () => ({ - processToolCall: vi.fn() + processToolCall: vi.fn(), + appendPendingToolImages: vi.fn() })) describe('convertOpenAIToAnthropicMessages', () => { @@ -139,6 +140,42 @@ describe('convertOpenAIToAnthropicMessages', () => { expect(content[1]).toMatchObject({ type: 'tool_use', id: 'tool_old', name: 'list_resources' }) }) + it('converts a user message with an image_url part to an Anthropic base64 image block', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'what is this?' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAABBBB' } } + ] + } as any + ] + + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + + expect(out).toHaveLength(1) + expect(out[0].role).toBe('user') + const content = out[0].content as any[] + expect(content[0]).toMatchObject({ type: 'text', text: 'what is this?' }) + expect(content[1]).toMatchObject({ + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAABBBB' } + }) + // The trailing block is the image — the ephemeral cache breakpoint may land on it + // (cache_control is valid on image blocks). + expect(content[1].cache_control).toEqual({ type: 'ephemeral' }) + }) + + it('keeps a plain string user message unchanged (no array wrapping)', () => { + // Non-trailing so the last-block cache_control wrapping doesn't obscure it. + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'just text' }, + { role: 'assistant', content: 'ok' } + ] + const { messages: out } = convertOpenAIToAnthropicMessages(messages) + expect(out[0].content).toBe('just text') + }) + it('caches a trailing tool result even when the prior turn used no captured content', () => { const messages: ChatCompletionMessageParam[] = [ { role: 'user', content: 'q' }, @@ -162,3 +199,21 @@ describe('convertOpenAIToAnthropicMessages', () => { }) }) }) + +describe('partialWebSearchQuery', () => { + it('extracts the query prefix from JSON cut mid-string', () => { + expect(partialWebSearchQuery('{"query": "latest stable Post')).toBe('latest stable Post') + }) + + it('decodes escapes and never includes a trailing half-escape', () => { + expect(partialWebSearchQuery('{"query": "say \\"hi\\" to')).toBe('say "hi" to') + // Cut right after the backslash: the escape pair is incomplete, so the + // extracted prefix must stop before it rather than corrupt the label. + expect(partialWebSearchQuery('{"query": "say \\')).toBe('say ') + }) + + it('returns undefined when no query string has started', () => { + expect(partialWebSearchQuery('{"que')).toBeUndefined() + expect(partialWebSearchQuery('{"query": ')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index b6e7e57a17..0c9f31e517 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -16,8 +16,46 @@ import type { MessageStream } from '@anthropic-ai/sdk/lib/MessageStream' import type { AIProviderModel } from '$lib/gen' import { getProviderAndCompletionConfig, workspaceAIClients } from '../lib' import { applyReasoningToConfig } from '../reasoningRegistry' -import { processToolCall, type Tool, type ToolCallbacks } from './shared' +import { + appendPendingToolImages, + processToolCall, + type Tool, + type ToolCallbacks, + type WebSearchSource +} from './shared' import { anthropicUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' +import { parseImageDataUrl } from './imageUtils' + +const ANTHROPIC_IMAGE_MEDIA_TYPES = new Set(['image/jpeg', 'image/png', 'image/gif', 'image/webp']) + +/** + * Convert an OpenAI user-message content array (text + image_url parts) to Anthropic + * content blocks. Returns a plain string when the content is a lone text part so + * simple messages stay unchanged. Non-image/text parts are dropped. + */ +function openAIUserContentToAnthropic(content: unknown): string | any[] { + if (typeof content === 'string') return content + if (!Array.isArray(content)) return JSON.stringify(content) + const blocks: any[] = [] + for (const part of content) { + if (part?.type === 'text' && typeof part.text === 'string') { + blocks.push({ type: 'text', text: part.text }) + } else if (part?.type === 'image_url' && part.image_url?.url) { + const { mediaType, base64 } = parseImageDataUrl(part.image_url.url) + if (!base64) continue + blocks.push({ + type: 'image', + source: { + type: 'base64', + media_type: ANTHROPIC_IMAGE_MEDIA_TYPES.has(mediaType) ? mediaType : 'image/png', + data: base64 + } + }) + } + } + if (blocks.length === 1 && blocks[0].type === 'text') return blocks[0].text + return blocks +} interface ParsedCompletionResult { shouldContinue: boolean @@ -30,23 +68,63 @@ function setAnthropicWebSearchStatus( callbacks: ToolCallbacks & { onMessageEnd: () => void }, toolId: string, status: WebSearchStatus, - errorCode?: string + details?: { errorCode?: string; query?: string; sources?: WebSearchSource[] } ) { const isLoading = status === 'searching' const failed = status === 'failed' + const sources = details?.sources callbacks.onMessageEnd() callbacks.setToolStatus(`anthropic_web_search:${toolId}`, { - content: failed ? 'Web search failed' : isLoading ? 'Searching the web...' : 'Searched the web', - error: failed ? `Web search failed${errorCode ? `: ${errorCode}` : ''}` : undefined, + content: failed + ? 'Web search failed' + : isLoading + ? details?.query + ? `Searching the web for "${details.query}"...` + : 'Searching the web...' + : details?.query + ? `Searched the web for "${details.query}"` + : 'Searched the web', + error: failed + ? `Web search failed${details?.errorCode ? `: ${details.errorCode}` : ''}` + : undefined, isLoading, isStreamingArguments: false, needsConfirmation: false, toolName: 'web_search', - showDetails: false, - autoCollapseDetails: true + // Sources keep the card expanded (no auto-collapse) so the consulted + // pages surface live as each search completes mid-stream. + ...(sources?.length + ? { webSearchSources: sources, showDetails: true, autoCollapseDetails: false } + : {}) }) } +// The query streams as partial JSON ({"query": "..."} cut mid-string), so +// JSON.parse fails until the block completes; regex out the string prefix to +// label the card while the model is still typing the query. +export function partialWebSearchQuery(partialJson: string): string | undefined { + const m = partialJson.match(/"query"\s*:\s*"((?:[^"\\]|\\.)*)/) + if (!m || !m[1]) return undefined + try { + return JSON.parse(`"${m[1]}"`) + } catch { + return undefined + } +} + +function anthropicWebSearchSources( + content: Anthropic.Messages.WebSearchToolResultBlock['content'] +): { errorCode?: string; sources?: WebSearchSource[] } { + if (!Array.isArray(content)) { + return { errorCode: content?.error_code } + } + return { + sources: content + .filter((r) => r.type === 'web_search_result') + .map((r) => ({ url: r.url, title: r.title })) + } +} + export async function getAnthropicCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -121,9 +199,16 @@ export async function parseAnthropicCompletion( let error = null let currentStreamingTool: - | { tempId: string; shouldStream: boolean; toolName: string } + | { tempId: string; shouldStream: boolean; toolName: string; isWebSearch?: boolean } | undefined = undefined let accumulatedJson = '' + // server_tool_use id → query, filled while the query streams; the paired + // web_search_tool_result block only carries tool_use_id. + const webSearchQueries = new Map() + // Result blocks already surfaced from stream events: the end-of-turn message + // handler must not re-emit these — re-setting the status would re-expand a + // card the user collapsed in the meantime. + const surfacedWebSearchResults = new Set() completion.on('streamEvent', (event: RawMessageStreamEvent) => { if (event.type === 'content_block_start') { @@ -143,7 +228,7 @@ export async function parseAnthropicCompletion( callbacks.setToolStatus(toolId, { isLoading: true, - content: `Calling ${toolName}...`, + content: tool?.streamingLabel ?? `Calling ${toolName}...`, toolName, isStreamingArguments: shouldStream, showFade: tool?.showFade, @@ -151,7 +236,26 @@ export async function parseAnthropicCompletion( autoCollapseDetails: tool?.autoCollapseDetails }) } else if (block.type === 'server_tool_use' && block.name === 'web_search') { + accumulatedJson = '' + currentStreamingTool = { + tempId: block.id, + shouldStream: false, + toolName: 'web_search', + isWebSearch: true + } setAnthropicWebSearchStatus(callbacks, block.id, 'searching') + } else if (block.type === 'web_search_tool_result') { + // Server tool results arrive complete in content_block_start; surface + // the source list now, while the model is still streaming the rest of + // its turn, instead of waiting for the end-of-turn message event. + surfacedWebSearchResults.add(block.tool_use_id) + const { errorCode, sources } = anthropicWebSearchSources(block.content) + setAnthropicWebSearchStatus( + callbacks, + block.tool_use_id, + errorCode ? 'failed' : 'completed', + { errorCode, query: webSearchQueries.get(block.tool_use_id), sources } + ) } } }) @@ -172,6 +276,15 @@ export async function parseAnthropicCompletion( }) completion.on('inputJson', (partialJson: string) => { + if (currentStreamingTool?.isWebSearch) { + accumulatedJson += partialJson + const query = partialWebSearchQuery(accumulatedJson) + if (query) { + webSearchQueries.set(currentStreamingTool.tempId, query) + setAnthropicWebSearchStatus(callbacks, currentStreamingTool.tempId, 'searching', { query }) + } + return + } if (currentStreamingTool?.shouldStream && currentStreamingTool.tempId) { // Accumulate the partial JSON accumulatedJson += partialJson @@ -201,6 +314,16 @@ export async function parseAnthropicCompletion( }) completion.on('message', (message: Message) => { + // Final message blocks carry the complete query; overwrite whatever the + // partial-JSON extraction reconstructed during streaming. + for (const block of message.content) { + if (block.type === 'server_tool_use' && block.name === 'web_search') { + const query = (block.input as any)?.query + if (typeof query === 'string' && query.length > 0) { + webSearchQueries.set(block.id, query) + } + } + } for (const block of message.content) { if (block.type === 'text') { const text = block.text @@ -208,13 +331,17 @@ export async function parseAnthropicCompletion( messages.push(assistantMessage) addedMessages.push(assistantMessage) callbacks.onMessageEnd() - } else if (block.type === 'web_search_tool_result') { - const errorCode = Array.isArray(block.content) ? undefined : block.content.error_code + } else if ( + block.type === 'web_search_tool_result' && + !surfacedWebSearchResults.has(block.tool_use_id) + ) { + // Fallback for a result whose content_block_start was missed. + const { errorCode, sources } = anthropicWebSearchSources(block.content) setAnthropicWebSearchStatus( callbacks, block.tool_use_id, errorCode ? 'failed' : 'completed', - errorCode + { errorCode, query: webSearchQueries.get(block.tool_use_id), sources } ) } else if (block.type === 'tool_use') { // Convert Anthropic tool calls to OpenAI format for compatibility @@ -310,6 +437,7 @@ export async function parseAnthropicCompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) return { shouldContinue: true, tokenUsage } } @@ -367,8 +495,7 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage if (message.role === 'user') { anthropicMessages.push({ role: 'user', - content: - typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + content: openAIUserContentToAnthropic(message.content) }) } else if (message.role === 'assistant') { // Replay a captured assistant turn verbatim so its thinking-block signatures @@ -443,8 +570,8 @@ export function convertOpenAIToAnthropicMessages(messages: ChatCompletionMessage // block of the last message. Each continuation only appends a tool result plus the // next turn, so everything up to here is read from cache — which is what keeps // replaying assistant turns verbatim (web-search results included) affordable. - // cache_control is valid on text/tool_use/tool_result blocks, but a thinking or - // redacted_thinking block must never be modified, so skip the breakpoint there. + // cache_control is valid on text/tool_use/tool_result/image blocks, but a thinking + // or redacted_thinking block must never be modified, so skip the breakpoint there. if (anthropicMessages.length > 0) { const lastMessage = anthropicMessages[anthropicMessages.length - 1] if (typeof lastMessage.content === 'string') { diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 599a77e906..b4fae2649a 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -924,7 +924,17 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy ## App Structure ### Frontend -- The frontend is bundled using esbuild with entrypoint \`index.tsx\` +- The frontend is bundled using esbuild, with entrypoint \`index.tsx\` for React and \`index.ts\` for Svelte and Vue +- The entrypoint is also the **mount** entrypoint: nothing is auto-rendered, so it must mount a top-level \`App\` into \`#root\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim: + \`\`\`tsx + import React from 'react' + import { createRoot } from 'react-dom/client' + import App from './App' + + createRoot(document.getElementById('root')!).render() + \`\`\` + (Svelte \`index.ts\`: \`mount(App, { target: document.getElementById('root')! })\`; Vue \`index.ts\`: \`createApp(App).mount('#root')\`.) +- **Never replace the entrypoint with a bare component.** A component that is defined but never mounted renders a blank screen with **no error** — it never executes, so nothing throws. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. - Frontend files are managed separately from backend runnables - The \`wmill.d.ts\` file is generated automatically from the backend runnables shape - Begin every React file (\`.tsx\`/\`.jsx\`) that uses JSX with \`import React from 'react'\`. Raw apps bundle with the classic JSX transform, so \`React\` must be in scope wherever JSX is used — a missing import compiles fine but throws \`React is not defined\` at runtime. diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte new file mode 100644 index 0000000000..2aadcca63d --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactViewer.svelte @@ -0,0 +1,97 @@ + + +
+
+
+ + + {artifact.name} + +
+
+ + + {#if canPreview} + (showSource = v === 'source')} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} +
+
+ +
+ {#if source} + + {#key `${artifact.id}:${artifact.updatedAt}`} + + {/key} + {:else} + +
+
+ +
+ {/if} +
+
diff --git a/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte new file mode 100644 index 0000000000..96412d9b3c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/artifacts/ArtifactsSegment.svelte @@ -0,0 +1,64 @@ + + + a.id} + rowTitle={(a) => a.name} + onPick={(a: PersistedArtifact) => aiChatManager.openArtifact?.(a.id, a.name)} +> + {#snippet row(a)} + {a.name} + + {a.kind} + + + + + {/snippet} + {#snippet actions(a)} + ', + uiMessage: 'Searched app DOM', + toolResult: 'match' + })) + setGetDomHandler(handler) + const result = await callGlobalTool( + 'search_dom', + { selector: 'button', pattern: 'Go', ignore_case: true }, + toolCallbacks, + { sessionId: 'sess-dom' } + ) + expect(result).toContain('Found 1 matching line(s)') + expect(handler).toHaveBeenCalledWith({ + sessionId: 'sess-dom', + query: { mode: 'search', selector: 'button', pattern: 'Go', ignoreCase: true } + }) + }) + + it('dispatches read_dom to the handler with a read query (whole-page when no selector)', async () => { + const handler = vi.fn(async () => ({ + aiResult: 'Live DOM for whole page (): Showing lines 1-1 of 1.', + uiMessage: 'Read app DOM', + toolResult: 'dom' + })) + setGetDomHandler(handler) + await callGlobalTool('read_dom', { start_line: 2, end_line: 40 }, toolCallbacks, { + sessionId: 'sess-dom' + }) + expect(handler).toHaveBeenCalledWith({ + sessionId: 'sess-dom', + query: { mode: 'read', selector: undefined, startLine: 2, endLine: 40 } + }) + }) + }) }) describe('session-only preview tools gating', () => { @@ -3619,6 +4286,8 @@ describe('session-only preview tools gating', () => { expect(names).not.toContain('get_preview_status') expect(names).not.toContain('get_app_runtime_logs') expect(names).not.toContain('list_app_runs') + expect(names).not.toContain('search_dom') + expect(names).not.toContain('read_dom') // other tools are still present expect(names).toContain('write_script') }) @@ -3629,8 +4298,29 @@ describe('session-only preview tools gating', () => { expect(names).toContain('get_preview_status') expect(names).toContain('get_app_runtime_logs') expect(names).toContain('list_app_runs') - // session set is the full globalTools - expect(names.length).toBe(globalTools.length) + expect(names).toContain('search_dom') + expect(names).toContain('read_dom') + // The session set is the full globalTools minus capability-gated tools: + // this environment is not Chromium, so take_screenshot is withheld (DOM + // capture is only faithful on Blink). search_dom / read_dom are not gated. + expect(names).not.toContain('take_screenshot') + expect(names.length).toBe(globalTools.length - 1) + }) + + it('offers take_screenshot inside a session only on Chromium', () => { + vi.stubGlobal('navigator', { + userAgentData: { brands: [{ brand: 'Chromium', version: '138' }] }, + userAgent: 'stubbed' + }) + try { + const names = toolNames(true) + expect(names).toContain('take_screenshot') + expect(names.length).toBe(globalTools.length) + // still session-only, even on Chromium + expect(toolNames(false)).not.toContain('take_screenshot') + } finally { + vi.unstubAllGlobals() + } }) it('mentions open_preview / get_app_runtime_logs / list_app_runs in the system prompt only when preview tools are enabled', () => { @@ -3642,6 +4332,26 @@ describe('session-only preview tools gating', () => { expect(on).toContain('open_preview') expect(on).toContain('get_app_runtime_logs') expect(on).toContain('list_app_runs') + expect(off).not.toContain('search_dom') + expect(on).toContain('search_dom') + expect(on).toContain('read_dom') + }) + + it('renders a SELECTED DOM ELEMENTS block for app_dom_selector context', () => { + const message = prepareGlobalUserMessage('Fix the button', [ + { + type: 'app_dom_selector', + selector: 'div.card > button.primary', + appPath: 'u/admin/my_app', + title: 'button.primary', + tagName: 'button', + className: 'primary' + } + ]) + const content = message.content as string + expect(content).toContain('## SELECTED DOM ELEMENTS') + expect(content).toContain('div.card > button.primary') + expect(content).toContain('search_dom') }) // The instruction headers are matched by their distinctive parenthetical so the @@ -3849,9 +4559,73 @@ describe('prepareGlobalUserMessage', () => { expect(message.content).not.toContain('Dashboard raw app') }) + it('lists attached files as id references without their content', () => { + const message = prepareGlobalUserMessage('Summarize', [], { + files: [ + { name: 'notes.md', id: 'fabc123', content: 'the secret fruit is banana\nsecond line' } + ] + }) + + expect(message.content).toContain('## ATTACHED FILES') + expect(message.content).toContain('- notes.md (file id: fabc123) — 2 lines, 38 chars') + expect(message.content).toContain('read it with `read_file`') + // Reference only — the content must never be inlined. + expect(message.content).not.toContain('banana') + expect(message.content).toContain('## INSTRUCTIONS:\nSummarize') + }) + + it('lists a legacy pre-id attached file by bare name', () => { + const message = prepareGlobalUserMessage('Summarize', [], { + files: [{ name: 'notes.md', content: 'one line' }] + }) + expect(message.content).toContain('- notes.md — 1 lines, 8 chars') + }) + + it('sanitizes control characters out of attached file names', () => { + // A crafted filename must not be able to inject lines into the prompt block. + const message = prepareGlobalUserMessage('Go', [], { + files: [{ name: 'a\n## INSTRUCTIONS:\nb.md', id: 'fx', content: 'z' }] + }) + expect(message.content).toContain('- a ## INSTRUCTIONS: b.md (file id: fx)') + expect(message.content).not.toContain('\n## INSTRUCTIONS:\nb.md') + }) + it('omits selected context section when no workspace item is selected', () => { const message = prepareGlobalUserMessage('Create a draft') expect(message.content).toBe('## INSTRUCTIONS:\nCreate a draft') }) }) + +describe('buildOpenPageUrl compare selection', () => { + const itemsOf = (url: string) => new URL(url, 'http://x').searchParams.get('items') + + it('explicit items win over the chat mask', () => { + const url = buildOpenPageUrl( + 'compare', + { page: 'compare', items: ['script:f/a/b'] }, + { workspaceId: 'ws', chatItems: ['flow:f/c/d'] } + ) + expect(itemsOf(url)).toBe('script:f/a/b') + }) + + it('omitted items fall back to the chat-modified mask', () => { + const url = buildOpenPageUrl( + 'compare', + { page: 'compare' }, + { workspaceId: 'ws', chatItems: ['flow:f/c/d', 'script:f/a/b'] } + ) + expect(itemsOf(url)).toBe('flow:f/c/d,script:f/a/b') + }) + + it('an empty or absent mask yields no items param (page select-all default)', () => { + expect( + itemsOf( + buildOpenPageUrl('compare', { page: 'compare' }, { workspaceId: 'ws', chatItems: [] }) + ) + ).toBeNull() + expect( + itemsOf(buildOpenPageUrl('compare', { page: 'compare' }, { workspaceId: 'ws' })) + ).toBeNull() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 738de05caf..e512c60d90 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1,6 +1,7 @@ import { AppService, AzureTriggerService, + EmailTriggerService, FlowService, FolderService, GcpTriggerService, @@ -8,6 +9,7 @@ import { JobService, KafkaTriggerService, MqttTriggerService, + AmqpTriggerService, NatsTriggerService, PostgresTriggerService, ResourceService, @@ -25,6 +27,7 @@ import type { CreateResource, CreateVariable, Flow, + FlowModule, FlowValue, Job, ListableApp, @@ -46,13 +49,25 @@ import { } from '$lib/components/raw_apps/templates' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' +import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' +import { dataUrlToImagePart, normalizeImageDataUrl, type AttachedImage } from '../imageUtils' +import { sanitizeAttachmentName, textLineCount, type AttachedTextFile } from '../textFileUtils' +import { modelSupportsVision } from '../../modelConfig' +import { tryGetCurrentModel } from '$lib/aiStore' +import { isChromiumBrowser } from '$lib/utils' import { applyEditableFlowJsonToFlow, buildEditableFlowJson, type EditableFlowJson, + finalizeUnresolvedInlineScripts, + restoreSpecialRawscriptModule, validateEditableFlowJson } from '../flow/editableFlowJson' -import { createInlineScriptSession } from '../flow/inlineScriptsUtils' +import { + createInlineScriptSession, + findUnresolvedInlineScriptRefs +} from '../flow/inlineScriptsUtils' +import { searchNpmPackagesTool } from '../script/core' import { getDatatableSdkReference, getFlowPrompt, @@ -73,15 +88,20 @@ import { executeTestRun, findAndReplace, type CreatedResourceTriggerKind, + type PreviewCardKind, type Tool, type ToolCallbacks, type ToolDisplayAction } from '../shared' import { searchDocsTool, readDocsPageTool } from '../docs/core' +import { createDbSchemaTool } from '../script/core' import type { ContextElement } from '../context' import { getDatatableTools } from '../datatableTools' +import { getDucklakeTools } from '../ducklakeTools' import { fileTools } from '../files/fileTools' import type { AttachedFilesStore } from '../files/attachedFiles.svelte' +import { artifactTools } from '../artifacts/artifactTools' +import type { SessionArtifactsStore } from '../artifacts/artifactsState.svelte' import { UserDraft } from '$lib/userDraft.svelte' import { emptySchema } from '$lib/utils' import { inferArgs } from '$lib/infer' @@ -111,8 +131,15 @@ import { workspaceStore } from '$lib/stores' import { get } from 'svelte/store' -import { deployDraft as deployDraftToWorkspace } from '$lib/utils_draft_deploy' +import { + canonicalDraftSideValue, + deployDraft as deployDraftToWorkspace, + getDraftDiffValues +} from '$lib/utils_draft_deploy' +import { changedLineIndices, draftDeployedPatch, windowPatch } from './draftDiff' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' +import type { UserDraftItemKind } from '$lib/gen' import { bundleRawAppDraft } from './rawAppBundlerBridge' import { buildRunsUrl, @@ -125,8 +152,13 @@ import { buildFoldersUrl, buildGroupsUrl, buildTriggersUrl, + buildCompareUrl, WORKSPACE_SETTINGS_TABS } from './pageNavigation' +import { + COMPARE_ITEMS_PARAM, + parseItemsMaskParam +} from '$lib/components/sessions/modifiedItemsMask' import { pageHref, TRIGGER_PAGES, @@ -135,6 +167,7 @@ import { import { clearEphemeralSecretVariableDraftValue, deleteGlobalDraft, + flushGlobalDraftSaves, getEphemeralSecretVariableDraftValue, getGlobalDraft, getGlobalDraftStoragePath, @@ -142,10 +175,33 @@ import { listGlobalDrafts, persistGlobalDraft, readGlobalDraftValue, + readLocalDraftCellByKind, + resolveGlobalDraftStoragePathByKind, saveGlobalAppDraft, setEphemeralSecretVariableDraftValue, type DraftPersistResult } from './userDraftAdapter' +import { + computeDiffParts, + expireWorkspaceDiffList, + getForkComparisonStatus, + getForkDiffIndex, + getForkParentWorkspaceId, + getWorkspaceDiffIndex, + maskVariableDiffSides, + readForkDiffEntries, + readWorkspaceDiffEntry, + resolveWorkspaceDiffTarget, + type DiffFileView, + type ForkDiffEntryView, + type WorkspaceDiffEntryView +} from './diffSnapshot' + +const VARIABLE_MASKED_NOTE = + 'Note: variable values are never shown in chat — the diff marks whether the value changed without revealing it.\n\n' +const SECRET_UNCOMPARABLE_NOTE = + 'Note: this is a SECRET variable — its value is never shown and cannot be compared, so it may ALSO have changed beyond what this diff shows.\n\n' +import { apiCatalogTools } from './apiCatalogTools' const ITEM_TYPES = [ 'script', @@ -190,6 +246,11 @@ export type GlobalActiveEditorContext = { export type GlobalUserMessageOptions = { workspace?: string activeEditor?: GlobalActiveEditorContext + /** Images attached to this message; delivered as image_url content parts. */ + images?: AttachedImage[] + /** Text files attached to this message; listed by reference below — the model + * reads their content on demand via the file tools. */ + files?: AttachedTextFile[] } const itemTypeSchema = z.enum(ITEM_TYPES) @@ -283,7 +344,17 @@ const listWorkspaceItemsSchema = z.object({ .min(1) .max(MAX_LIST_LIMIT) .optional() - .describe('Maximum number of items to return. Defaults to 50 and is capped at 100.') + .describe( + 'Maximum items per item type per page (for triggers, per trigger kind). Defaults to 50 and is capped at 100.' + ), + page: z + .number() + .int() + .min(1) + .optional() + .describe( + 'Page number, starting at 1. Each item type pages independently: request the next page while any type still returns a full page. Drafts appear on page 1 only, capped at limit per type.' + ) }) const readWorkspaceItemSchema = z.object({ @@ -291,7 +362,13 @@ const readWorkspaceItemSchema = z.object({ path: z.string().describe('Workspace path of the item to read.'), trigger_kind: triggerKindSchema .optional() - .describe('Required when type is trigger. Identifies which trigger service to call.') + .describe('Required when type is trigger. Identifies which trigger service to call.'), + version: z + .enum(['deployed']) + .optional() + .describe( + 'Pass "deployed" to read the deployed workspace state even when a draft exists (e.g. to learn the deployed input schema before running the deployed version). Default reads your draft when one exists.' + ) }) const draftOverrideField = z @@ -375,6 +452,10 @@ const writeFlowSchema = z.object({ override: draftOverrideField }) +// modules/preprocessor_module/failure_module can carry rawscript `content`, whose +// quotes and newlines are the usual reason the JSON string fails to parse. +const FLOW_CODE_BEARING_FIELDS = new Set(['modules', 'preprocessor_module', 'failure_module']) + function parseOptionalJsonArg(value: unknown, field: string): unknown { if (value === undefined || value === null) { return value @@ -384,10 +465,60 @@ function parseOptionalJsonArg(value: unknown, field: string): unknown { return typeof value === 'string' ? JSON.parse(value) : value } catch (error) { const message = error instanceof Error ? error.message : String(error) - throw new Error(`Invalid JSON for ${field}: ${message}`) + const hint = FLOW_CODE_BEARING_FIELDS.has(field) + ? ' A rawscript "content" string with multi-line code or quotes is the usual cause. Instead of inlining large code, create the rawscript module with empty content ("") and fill its body with set_flow_module_code afterwards.' + : '' + throw new Error(`Invalid JSON for ${field}: ${message}${hint}`) } } +/** + * Rawscript bodies are fragile to embed inside the `modules` JSON string: the + * code's quotes and newlines have to survive three levels of escaping (tool-call + * arguments -> modules string -> content string) and the model routinely mangles + * them. So a module may be saved with empty (or `inline_script.` placeholder) + * content; return the ids that still need a body filled out-of-band with + * `set_flow_module_code`. + */ +function emptyInlineScriptModuleIds(editable: EditableFlowJson): string[] { + const value: FlowValue = { + modules: editable.modules, + preprocessor_module: editable.preprocessor_module ?? undefined, + failure_module: editable.failure_module ?? undefined + } + const session = createInlineScriptSession() + buildEditableFlowJson({ value, schema: editable.schema }, session) + return Object.entries(session.getAll()) + .filter(([, content]) => content.trim() === '' || /^inline_script\./.test(content)) + .map(([id]) => id) +} + +/** + * Fold the empty-body warning into a flow write tool's JSON result — but only + * when the save actually succeeded. `writeFlowDraft` reports + * conflicts/persistence errors as `{ success: false }` rather than throwing; + * telling the model to fill code on a flow that was never saved would send it + * after a stale or nonexistent draft. + */ +function appendEmptyInlineScriptWarning(result: string, editable: EditableFlowJson): string { + const emptyIds = emptyInlineScriptModuleIds(editable) + if (emptyIds.length === 0) { + return result + } + let parsed: { success?: unknown; message?: unknown } + try { + parsed = JSON.parse(result) + } catch { + return result + } + if (parsed.success !== true || typeof parsed.message !== 'string') { + return result + } + const list = emptyIds.map((id) => `"${id}"`).join(', ') + parsed.message += `\n\nWarning: inline scripts ${list} have no code yet. Fill each one with set_flow_module_code(path, module_id, code) — do not re-send the whole flow.` + return JSON.stringify(parsed, null, 2) +} + function editableFlowToDraftValue(editable: EditableFlowJson): FlowDraftValue { const value: FlowValue = { modules: editable.modules, @@ -428,9 +559,11 @@ const writeTriggerSchema = z.object({ triggerRequestSchemas.nats, triggerRequestSchemas.postgres, triggerRequestSchemas.mqtt, + triggerRequestSchemas.amqp, triggerRequestSchemas.sqs, triggerRequestSchemas.gcp, - triggerRequestSchemas.azure + triggerRequestSchemas.azure, + triggerRequestSchemas.email ]) .describe( 'Full trigger configuration. Must include path, script_path, is_flow plus the kind-specific fields.' @@ -443,7 +576,11 @@ const writeResourceSchema = resourceRequestSchema.extend({ override: draftOverri const writeVariableSchema = variableRequestSchema.extend({ override: draftOverrideField }) const searchResourceTypesSchema = z.object({ - query: z.string().describe('Substring to match against resource type names.'), + query: z + .string() + .describe( + 'Natural-language description of the integration or capability you need, e.g. "stripe", "postgres database", or "send emails". Matched semantically against resource type names and descriptions, so describe the intent rather than guessing the exact name.' + ), limit: z .number() .int() @@ -520,6 +657,73 @@ const rebaseDraftSchema = z.object({ .describe('Workspace path of the draft to rebase onto the latest deployed version.') }) +const diffSchema = z.object({ + against: z + .enum(['deployed', 'parent_workspace']) + .optional() + .describe( + "What to compare against. 'deployed' (default): the current draft vs the deployed version. 'parent_workspace': the deployed fork vs its parent workspace (only in a fork; local drafts are flagged but not part of that comparison)." + ), + type: itemTypeSchema + .optional() + .describe('With path: the item to diff. Omit both type and path for the workspace index.'), + path: z + .string() + .optional() + .describe( + 'Workspace path of the item to diff (draft vs deployed). Omit for the index of every draft in the workspace.' + ), + trigger_kind: triggerKindSchema + .optional() + .describe('Required when type is trigger. Must match the draft trigger kind.'), + file: z + .string() + .optional() + .describe( + 'Item mode, multi-file apps only: read one file\'s diff inside the app (e.g. "src/App.tsx"). Omit for the per-file summary plus config changes.' + ), + search: z + .string() + .optional() + .describe( + 'Search mode: literal substring (case-insensitive, not a regex) matched against added/removed diff lines across every diff in the comparison. Ignores type/path.' + ), + file_glob: z + .string() + .optional() + .describe( + 'Search mode: optional glob filter on item paths and app file paths (e.g. "*.ts" matches file names, "f/dash/**" matches full paths).' + ), + max_matches: z + .number() + .int() + .min(1) + .optional() + .describe('Search mode: maximum matching diff lines returned (default 50, hard cap 200).'), + types: z + .array(itemTypeSchema) + .optional() + .describe('Index mode: only list drafts of these item types.'), + path_prefix: z + .string() + .optional() + .describe('Index mode: only list drafts under this path prefix, such as f/billing/.'), + offset: z + .number() + .int() + .min(0) + .optional() + .describe('Item mode: skip this many patch lines (paginate a large diff).'), + limit: z + .number() + .int() + .min(1) + .optional() + .describe( + 'Index mode: max items to list (default 50, capped at 100). Item mode: max patch lines to return (default 500).' + ) +}) + const editScriptSchema = z.object({ path: z.string().describe('Workspace path of the script to edit.'), old_string: z.string().min(1).describe('Exact text to find in the script source.'), @@ -803,6 +1007,44 @@ const listAppRunsSchema = z.object({ .describe('How many of the most recent backend runs to return, newest first. Defaults to 20.') }) +const domSelectorField = z + .string() + .optional() + .describe( + 'CSS selector for the element to inspect in the live raw app preview. Omit to target the whole page (). Prefer a selector from a DOM element chip the user attached. If it matches several elements, the first is used.' + ) + +const domAppPathField = z + .string() + .optional() + .describe( + "Raw-app path of the element, from its `app_path` in the SELECTED DOM ELEMENTS block. Pass it so the RIGHT app's preview is read even if another preview tab is now visible; omit to use the currently active preview. If that app's preview has been closed, the tool says so." + ) + +const searchDomSchema = z.object({ + app_path: domAppPathField, + selector: domSelectorField, + pattern: z.string().describe('JavaScript regular expression to search the rendered HTML for.'), + ignore_case: z.boolean().optional().describe('Case-insensitive matching. Defaults to false.') +}) + +const readDomSchema = z.object({ + app_path: domAppPathField, + selector: domSelectorField, + start_line: z + .number() + .int() + .optional() + .describe('1-based first line of the pretty-printed HTML to read. Defaults to 1.'), + end_line: z + .number() + .int() + .optional() + .describe('1-based last line to read. The window is capped at 200 lines.') +}) + +const takeScreenshotSchema = z.object({}) + const FRAMEWORK_KEYS = [ 'react19', 'react18', @@ -893,6 +1135,14 @@ const buildGlobalSystemPrompt = ( ) => { const folderGuidance = buildFolderGuidance(username, folderCtx) const folderGuidanceBlock = folderGuidance ? `\n${folderGuidance}` : '' + // `previewTools` doubles as "this is a session chat" — sessions are the only + // chats that get the preview tool set. The alpha heads-up only makes sense + // there, where the chat actively builds the pipeline on the canvas; the + // standalone global chat keeps its plain guidance. + const pipelineAlphaNote = previewTools + ? ' Data pipeline support in this chat is in ALPHA: the first time the user asks for a data pipeline in this session, briefly tell them it is an alpha feature before you start building.' + : '' + const pipelineBullet = `- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow.${pipelineAlphaNote}` return `You are Windmill's global workspace assistant. The current user's workspace username is "${username}". @@ -912,13 +1162,24 @@ Rules: - If the user message includes an ACTIVE EDITOR section, treat it as the currently open item and use it for references like "this", "current", or "open editor". - Use deploy_workspace_item only after the user explicitly asks to deploy. It persists a draft to the workspace. - Use discard_local_draft to remove a draft, including the matching open editor draft. Use delete_workspace_item only to delete a deployed workspace item. +- Use diff to review changes — before deploying, or when the user asks what changed. It is read-only: without arguments it lists every draft in the workspace with its change status; with type+path it returns that item's unified diff (for multi-file apps, pass file to read one file's diff). In a fork, pass against="parent_workspace" to compare the deployed fork with its parent workspace instead. Pass search to grep changed lines across all diffs. - Variable values are never readable. For secrets, create a secret variable and reference it from resources as "$var:path/to/variable". - Use search_resource_types before write_resource. +- When script or raw app code needs an external npm package you are not fully familiar with, use search_npm_packages to find it and get its documentation and type definitions. Link the package documentation in your answer when you rely on it. +- Use get_db_schema with a database resource path to fetch its tables and columns before writing SQL (or a script querying that database). - Use get_instructions before writing scripts, flows, resources, or apps. For scripts, pass the target language. -- A "data pipeline" is NOT a flow: it is a DAG of independent scripts in one folder, wired by storage assets (DuckLake/data tables/S3) and triggers via top-of-file \`pipeline\` / \`on \` annotation comments written in each script's comment syntax (\`--\` for SQL, \`#\` for Python/Bash, \`//\` for TS — a \`//\` line in a SQL node is a syntax error). When the user asks for a data pipeline (or to ingest/transform/materialize data across steps), call get_instructions with subject "pipeline" and build annotated script drafts — do not build a flow. +${pipelineBullet} - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_job_logs with a returned id to inspect a specific run's logs — without starting a new test run. - Use open_page to show a workspace page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, or Workspace settings on a specific tab (e.g. "open the failed runs of f/foo/bar", "open the schedule for X", "open the git sync settings"). Only the pages listed for this user in the tool are available; don't offer pages that aren't listed. Don't use it as a substitute for list_runs when you just need the data yourself. +- Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its edit drawer, or the page's filters otherwise). Never just describe where to click. +- When the user is happy with the changes and wants to review or deploy them, use open_page with page "compare" — it opens the Compare & Deploy review page.${ + previewTools + ? ' By default it preselects the items this chat modified; pass items (":" entries) to control the selection' + : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' + }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed. +- For a Windmill operation no other tool covers (workers, queue state, a run's result or args, ...), use search_api_endpoints to find a REST endpoint, then call_api_get for reads or call_api_endpoint for mutations (the user is asked to confirm those). Always prefer a dedicated tool when one exists; endpoints for authoring or deleting scripts, flows, apps, schedules, resources, or variables are not available through the API catalog tools — use the draft tools and delete_workspace_item instead. +- runScriptByPath / runFlowByPath from the API catalog run the DEPLOYED version of an item. Use them only when the user explicitly asks to run the deployed version, and read the item with read_workspace_item version: "deployed" first so the arguments match the deployed input schema (a draft may have different inputs). To test something you are editing or just wrote, always use test_run_script, test_run_flow, or test_run_step — they run the draft. - When a required decision is ambiguous, use askUserQuestion with two to ten clear proposed answer strings instead of guessing. The user can also type a custom answer when none of the proposed answers fit. Set multiSelect: true only when the answers can genuinely co-apply and the user may pick several (not mutually exclusive). - When the user asks you to remember a lasting preference, always/never do something, or change/stop a behavior going forward, call update_user_instructions to persist it. It edits only the USER INSTRUCTIONS block (not WORKSPACE INSTRUCTIONS). Keep each instruction concise; do not use it for one-off requests scoped to the current task. - Keep context targeted.${ @@ -927,8 +1188,15 @@ Rules: - After writing or substantially editing a script / flow / app draft, show it via open_preview(kind, path) so the user sees the editor and live preview right next to the chat. First check whether it is already shown: if unsure, call get_preview_status. Only call open_preview (or offer to) when no preview is open or it is showing a different item — don't re-open a preview already showing the item you just edited. - Building a data pipeline: call open_preview(kind="pipeline", path="") as the FIRST step, before creating any node — this opens the pipeline editor the user reviews in. path is the folder, not an item; an empty or not-yet-created folder is fine (create_folder first if needed, then open it). Opening it registers build_pipeline_node / edit_pipeline_node — use ONLY those to add or change pipeline nodes, never write_script for a pipeline node — they apply directly as unsaved drafts on the canvas (no separate accept/reject step) that the user reviews and deploys. Do not write pipeline scripts without first opening the editor. - When debugging a running raw app, call get_app_runtime_logs to read the live preview's browser console output. It needs the raw app preview open (open_preview kind="raw_app"). +- To inspect what actually rendered in a running raw app (verify an edit landed on screen, diagnose a blank/empty or wrong view, answer "what's showing"), use search_dom (regex over the live HTML) and read_dom (a line-numbered window). Pass a \`selector\` to scope to an element — prefer the selector from a DOM element chip the user attached — or omit it for the whole page. When a chip lists an \`app_path\`, pass it too so the RIGHT app is read (several previews can be open; a query without \`app_path\` hits the visible one). The DOM is read live and is never in context; no match means the element isn't rendered. Both need the raw app preview open. - get_app_runtime_logs only shows the app's browser console. For the server-side logs of a backend runnable the app invoked (a backend. call), call list_app_runs to get that run's job_id from the live preview, then get_job_logs with it. Use this when a backend call errors or returns something unexpected. -- open_page opens its page as a tab in the side-panel preview next to the chat — the only way to show one of these pages there (open_preview only handles editable items). Changing filters on a page already open updates that same tab; only pass new_tab when the user explicitly asks for a separate tab.` +${ + isChromiumBrowser() + ? `- When the user raises how a raw app looks (something is off, or they want the design or layout improved), call take_screenshot to see what they are looking at before changing anything. Reach for it when the request is about appearance, not to review your own edits, which you can read back from the code. It needs the raw app preview open (open_preview kind="raw_app").` + : `- When the user raises how a raw app looks (something is off, or they want the design or layout improved) and their description alone isn't specific enough to pinpoint the problem, ask them to paste or drop a screenshot of it into the chat before changing anything.` +} +- open_page opens its page as a tab in the side-panel preview next to the chat — the only way to show one of these pages there (open_preview only handles editable items). Changing filters on a page already open updates that same tab; only pass new_tab when the user explicitly asks for a separate tab. +- create_artifact saves a persistent markdown document (a planning doc, design write-up, spec, or other longer structured output) shown in the session preview panel. Prefer it over a long inline reply for content the user will revisit; keep brief answers inline. To revise one, call list_artifacts then read_artifact for the current content, then update_artifact to overwrite it — never create a second artifact for the same document.` : '' } @@ -993,7 +1261,10 @@ function scriptToItem(script: Script | NewScript, includeValue: boolean): Worksp summary: script.summary, language: script.language, value: includeValue ? script.content : undefined, - isDraft: false + schema: includeValue ? (script as Script).schema : undefined, + // Listings with includeDraftOnly synthesize rows for editor drafts that + // have no deployed counterpart — label those honestly. + isDraft: (script as Script).draft_only ?? false } } @@ -1005,7 +1276,7 @@ function flowToItem(flow: Flow, includeValue: boolean): WorkspaceItem { value: includeValue ? { value: flow.value, schema: flow.schema, groups: flow.value.groups ?? null } : undefined, - isDraft: false + isDraft: (flow as Flow & { draft_only?: boolean }).draft_only ?? false } } @@ -1392,7 +1663,12 @@ function triggerToItem( type TriggerService = { exists(args: { workspace: string; path: string }): Promise get(args: { workspace: string; path: string }): Promise - list(args: { workspace: string; pathStart?: string; perPage?: number }): Promise + list(args: { + workspace: string + pathStart?: string + perPage?: number + page?: number + }): Promise create(args: { workspace: string; requestBody: any }): Promise update(args: { workspace: string; path: string; requestBody: any }): Promise delete(args: { workspace: string; path: string }): Promise @@ -1447,6 +1723,14 @@ const triggerServices: Record = { update: (a) => MqttTriggerService.updateMqttTrigger(a), delete: (a) => MqttTriggerService.deleteMqttTrigger(a) }, + amqp: { + exists: (a) => AmqpTriggerService.existsAmqpTrigger(a), + get: (a) => AmqpTriggerService.getAmqpTrigger(a), + list: (a) => AmqpTriggerService.listAmqpTriggers(a), + create: (a) => AmqpTriggerService.createAmqpTrigger(a), + update: (a) => AmqpTriggerService.updateAmqpTrigger(a), + delete: (a) => AmqpTriggerService.deleteAmqpTrigger(a) + }, sqs: { exists: (a) => SqsTriggerService.existsSqsTrigger(a), get: (a) => SqsTriggerService.getSqsTrigger(a), @@ -1470,6 +1754,14 @@ const triggerServices: Record = { create: (a) => AzureTriggerService.createAzureTrigger(a), update: (a) => AzureTriggerService.updateAzureTrigger(a), delete: (a) => AzureTriggerService.deleteAzureTrigger(a) + }, + email: { + exists: (a) => EmailTriggerService.existsEmailTrigger(a), + get: (a) => EmailTriggerService.getEmailTrigger(a), + list: (a) => EmailTriggerService.listEmailTriggers(a), + create: (a) => EmailTriggerService.createEmailTrigger(a), + update: (a) => EmailTriggerService.updateEmailTrigger(a), + delete: (a) => EmailTriggerService.deleteEmailTrigger(a) } } @@ -1477,18 +1769,27 @@ async function readWorkspaceItem( type: WorkspaceItemType, path: string, workspace: string, - triggerKind?: TriggerKind + triggerKind?: TriggerKind, + deployedOnly = false ): Promise { switch (type) { case 'script': { - // Prefer the DB draft (newer than the deployed version) when one exists. - const script = await ScriptService.getScriptByPath({ workspace, path, getDraft: true }) - return scriptToItem((script.draft as Script | undefined) ?? script, true) + // Prefer the DB draft (newer than the deployed version) when one exists, + // unless the caller explicitly asked for the deployed state. + const script = await ScriptService.getScriptByPath({ + workspace, + path, + getDraft: !deployedOnly + }) + const draft = deployedOnly ? undefined : (script.draft as Script | undefined) + return scriptToItem(draft ?? script, true) } case 'flow': { - // Prefer the DB draft (newer than the deployed version) when one exists. - const flow = await FlowService.getFlowByPath({ workspace, path, getDraft: true }) - return flowToItem((flow.draft as Flow | undefined) ?? flow, true) + // Prefer the DB draft (newer than the deployed version) when one exists, + // unless the caller explicitly asked for the deployed state. + const flow = await FlowService.getFlowByPath({ workspace, path, getDraft: !deployedOnly }) + const draft = deployedOnly ? undefined : (flow.draft as Flow | undefined) + return flowToItem(draft ?? flow, true) } case 'schedule': return scheduleToItem(await ScheduleService.getSchedule({ workspace, path }), true) @@ -1529,7 +1830,8 @@ async function listWorkspaceItems( types: WorkspaceItemType[], workspace: string, pathPrefix: string | undefined, - perPage: number + perPage: number, + page?: number ): Promise { const items: WorkspaceItem[] = [] @@ -1538,6 +1840,7 @@ async function listWorkspaceItems( workspace, pathStart: pathPrefix, perPage, + page, includeDraftOnly: true, withoutDescription: true }) @@ -1549,6 +1852,7 @@ async function listWorkspaceItems( workspace, pathStart: pathPrefix, perPage, + page, includeDraftOnly: true, withoutDescription: true }) @@ -1559,18 +1863,30 @@ async function listWorkspaceItems( const schedules = await ScheduleService.listSchedules({ workspace, pathStart: pathPrefix, - perPage + perPage, + page }) for (const schedule of schedules) items.push(scheduleToItem(schedule, false)) } if (types.includes('trigger')) { for (const kind of TRIGGER_KINDS) { - const triggers = await triggerServices[kind].list({ - workspace, - pathStart: pathPrefix, - perPage - }) + let triggers: Awaited> + try { + triggers = await triggerServices[kind].list({ + workspace, + pathStart: pathPrefix, + perPage, + page + }) + } catch (err) { + // A trigger kind whose backend routes aren't compiled in (e.g. email + // without smtp+private, or an EE kind on CE) 404s here; skip only that + // so one unavailable kind doesn't drop the whole listing. Any other + // failure (auth, 5xx, network) is real and must surface. + if ((err as { status?: number } | undefined)?.status === 404) continue + throw err + } for (const trigger of triggers) items.push(triggerToItem(kind, trigger, false)) } } @@ -1579,7 +1895,8 @@ async function listWorkspaceItems( const resources = await ResourceService.listResource({ workspace, pathStart: pathPrefix, - perPage + perPage, + page }) for (const resource of resources) items.push(resourceToItem(resource, false)) } @@ -1588,7 +1905,8 @@ async function listWorkspaceItems( const variables = await VariableService.listVariable({ workspace, pathStart: pathPrefix, - perPage + perPage, + page }) for (const variable of variables) items.push(variableToItem(variable)) } @@ -1597,7 +1915,8 @@ async function listWorkspaceItems( const apps = await AppService.listApps({ workspace, pathStart: pathPrefix, - perPage + perPage, + page }) for (const app of apps) items.push(appToItem(app, false)) } @@ -1656,7 +1975,9 @@ function getFlowInstructions(): string { - \`read_flow_module_code(path, module_id)\` — returns the raw inline script content for one module. - \`set_flow_module_code(path, module_id, code)\` — overwrites that module's inline script content; saves to the draft. - Use \`patch_flow_json\` for *structural* edits: module ids, paths, input_transforms, branch arrangement, summaries, preprocessor/failure swaps, schema/groups/notes. Use \`set_flow_module_code\` for changes inside a specific rawscript body. -- \`write_flow\` is for full overwrites / create-from-scratch. Its \`modules\`, \`preprocessor_module\`, and \`failure_module\` arguments use **non-compact** flow modules (rawscript content is the actual code, not a placeholder). +- When **adding a new rawscript module** with \`patch_flow_json\`, set its \`content\` to the placeholder \`"inline_script."\` (matching the compact view) — never real code — then **immediately fill the body** with \`set_flow_module_code(path, module_id, code)\`. The module is saved with an empty body until you do; the tool result lists the module ids still awaiting code. A placeholder that names anything other than the module's own id or an existing rawscript module is rejected. +- \`write_flow\` is for full overwrites / create-from-scratch. Its \`modules\`, \`preprocessor_module\`, and \`failure_module\` arguments are **non-compact** flow modules: inline each rawscript body directly in \`content\` by default. But if a body is long or quote/backslash-heavy enough that escaping it into the JSON string is error-prone — or if a \`write_flow\` call comes back with a JSON parse error — create that module with **empty content** (\`"content": ""\`) and fill it with \`set_flow_module_code(path, module_id, code)\`, whose \`code\` is a plain argument with no nested escaping. \`write_flow\` reports which modules still have empty bodies so you know what to fill. + - When overwriting an **existing** flow, set \`"content": "inline_script."\` on any rawscript module whose code you are not changing — the placeholder resolves to that module's current body, so you never re-send (or re-read) unchanged code. Placeholders that match no existing rawscript module and are not the module's own id are rejected. # Windmill flow authoring reference @@ -1739,6 +2060,71 @@ function getInstructions(subject: InstructionSubject, language?: ScriptLang): st export type AiSkillListItem = { name: string; description: string } +/** Live session facts appended to the GLOBAL system prompt for session chats. + * Provided by the session runtime as a resolver (copilot must not import the + * sessions modules) and re-read on every system-message rebuild — the fork + * commits at first send, and the user can re-point the session's workspace. */ +export type SessionPromptContext = { + /** Operating workspace (undefined while the session is an unsent draft with + * no pick). Only slug-validated workspace IDs belong here — free-form + * metadata like display names is user-controlled text that must not be + * interpolated into the system prompt. */ + workspaceId?: string + /** Set when the operating workspace is a fork of this workspace (staged + * session fork or persistent dev workspace — `isDevWorkspace` splits them). */ + parentWorkspaceId?: string + /** The operating workspace is a persistent dev workspace, not an ephemeral + * staged fork. Same promote-to-parent deploy flow; different lifecycle. */ + isDevWorkspace?: boolean + /** Committed workspace missing from the user's workspace list (access lost / + * stale store): still a fork per `isForkSession`, but the parent is unknown — + * must not be presented as the live workspace. */ + forkParentUnknown?: boolean + /** Pre-send intent: a staged fork of this workspace is created at first send. */ + pendingForkOf?: string +} + +/** Session-state guidance appended to the global system prompt so the model + * knows where its work lands (staged fork vs the live workspace). */ +export function getSessionContextPromptSection(ctx: SessionPromptContext): string { + const lines = [ + '', + '', + 'Session state:', + '- This chat is a Windmill AI session with its own operating workspace: every tool call (reads, drafts, test runs, deploys) targets that workspace.' + ] + if (ctx.pendingForkOf) { + lines.push( + `- No workspace is committed yet: a staged fork of workspace "${ctx.pendingForkOf}" is created automatically when the first message is sent, and all work lands in that fork.` + ) + } else if (ctx.parentWorkspaceId && ctx.isDevWorkspace) { + lines.push( + `- Operating workspace: "${ctx.workspaceId}" — the user's persistent DEV WORKSPACE, forked from workspace "${ctx.parentWorkspaceId}". deploy_workspace_item publishes into the dev workspace only; the user reviews & promotes changes into "${ctx.parentWorkspaceId}" from the session's deploy panel. Never present a change as live in "${ctx.parentWorkspaceId}".` + ) + } else if (ctx.parentWorkspaceId) { + lines.push( + `- Operating workspace: "${ctx.workspaceId}" — an ephemeral STAGED FORK of workspace "${ctx.parentWorkspaceId}", created for session work. deploy_workspace_item publishes into the fork only, and the user reviews & promotes fork changes into "${ctx.parentWorkspaceId}" from the session's deploy panel. Never present a change as live in "${ctx.parentWorkspaceId}".` + ) + } else if (ctx.forkParentUnknown) { + lines.push( + `- Operating workspace: "${ctx.workspaceId}" — a fork whose parent workspace is not currently visible to this user. deploy_workspace_item publishes into the fork only; the user promotes changes from the session's deploy panel. Never present a change as live in any other workspace.` + ) + } else if (ctx.workspaceId) { + lines.push( + `- Operating workspace: "${ctx.workspaceId}" — the live workspace itself, not a fork. deploy_workspace_item publishes directly to everyone in it.` + ) + } else { + lines.push( + '- No operating workspace is set yet; the user picks one (or a new staged fork) before the first message is sent.' + ) + } + return lines.join('\n') +} + +/** `/` picker entry: a workspace skill or a built-in session action. The kind + * drives the picker's category grouping; entries without one are ungrouped. */ +export type ChatCommandItem = AiSkillListItem & { kind?: 'action' | 'skill' } + /** Fetch the workspace's AI skills (name + description) for the global system prompt. */ export async function loadWorkspaceSkills(workspace: string): Promise { if (!workspace) return [] @@ -1790,7 +2176,8 @@ const OPEN_PAGE_NAMES = [ 'folders', 'groups', 'triggers', - 'workspace_settings' + 'workspace_settings', + 'compare' ] as const type OpenPageName = (typeof OPEN_PAGE_NAMES)[number] @@ -1804,7 +2191,8 @@ const OPEN_PAGE_LABELS: Record = { folders: 'Folders', groups: 'Groups', triggers: 'Triggers', - workspace_settings: 'Workspace settings' + workspace_settings: 'Workspace settings', + compare: 'Compare & Deploy' } // Trigger kinds available given the workspace's license — the EE-gated kinds @@ -1841,12 +2229,24 @@ function allowedOpenPages(workspaceId: string | undefined = get(workspaceStore)) 'audit_logs', 'folders', 'groups', - 'triggers' + 'triggers', + 'compare' ]) if (isAdmin) allowed.add('workspace_settings') return OPEN_PAGE_NAMES.filter((p) => allowed.has(p)) } +// The advertised `items` description must match this chat's surface: only chats that +// track their modified items (AI sessions) can honor "omitted = this chat's edits" — +// on an untracked chat (the global side panel) an omitted mask falls through to the +// page's select-all default, so the model is told to pass the items explicitly there. +const COMPARE_ITEMS_DESCRIPTIONS = { + tracked: + "Compare: preselect exactly these changed items, each as ':' where kind is script, flow, raw_app, app, resource, variable, or a trigger kind like trigger_schedule / trigger_http (e.g. 'script:f/foo/bar'). Omit to preselect the items modified in this chat (everything when this chat modified nothing).", + untracked: + "Compare: preselect exactly these changed items, each as ':' where kind is script, flow, raw_app, app, resource, variable, or a trigger kind like trigger_schedule / trigger_http (e.g. 'script:f/foo/bar'). If omitted, the page preselects EVERY pending change in the workspace, not just this chat's — when you changed specific items, pass them so the review is scoped to them." +} as const + // One flat object (not a discriminated union): `page` selects the target and the // per-page fields are optional. Top-level `type: object` is what Anthropic's // input_schema requires; a top-level oneOf would be rejected. Each per-page URL builder @@ -1854,20 +2254,7 @@ function allowedOpenPages(workspaceId: string | undefined = get(workspaceStore)) // apply to the chosen page is harmless. This full schema is used to PARSE tool args; the // advertised schema (what the model sees) is narrowed per-user in `setSchema`. const openPageFullSchema = z.object({ - page: z - .enum([ - 'runs', - 'schedules', - 'variables', - 'resources', - 'assets', - 'audit_logs', - 'folders', - 'groups', - 'triggers', - 'workspace_settings' - ]) - .describe('Which page to open'), + page: z.enum(OPEN_PAGE_NAMES).describe('Which page to open'), path: z .string() .optional() @@ -1893,7 +2280,7 @@ const openPageFullSchema = z.object({ .string() .optional() .describe( - 'Schedules/Triggers: exact schedule or trigger path to open in the edit drawer, e.g. f/foo/my_schedule' + 'Schedules/Triggers/Variables/Resources: exact item path to open in the edit drawer, e.g. f/foo/my_schedule. Use it whenever the user should act on one specific item (e.g. fill in credentials) so they land directly in its editor.' ), summary: z .string() @@ -1918,6 +2305,13 @@ const openPageFullSchema = z.object({ .enum([...WORKSPACE_SETTINGS_TABS] as [string, ...string[]]) .optional() .describe('Workspace settings: which settings tab to open'), + mode: z + .enum(['draft', 'fork']) + .optional() + .describe( + "Compare: which comparison to show — 'draft' (deployed items vs their pending drafts) or 'fork' (this forked workspace vs its parent). Omit to auto-pick: the view containing the preselected items (draft whenever any of them is a pending draft); with nothing preselected, fork on a forked workspace and draft otherwise." + ), + items: z.array(z.string()).min(1).optional().describe(COMPARE_ITEMS_DESCRIPTIONS.tracked), new_tab: z .boolean() .optional() @@ -1936,7 +2330,7 @@ const OPEN_PAGE_FIELD_PAGES: Record = { schedule_path: ['runs', 'schedules'], job_kinds: ['runs'], user: ['runs'], - open: ['schedules', 'triggers'], + open: ['schedules', 'triggers', 'variables', 'resources'], summary: ['schedules'], trigger_kind: ['triggers'], resource_type: ['resources'], @@ -1944,7 +2338,9 @@ const OPEN_PAGE_FIELD_PAGES: Record = { username: ['audit_logs'], operation: ['audit_logs'], resource: ['audit_logs'], - tab: ['workspace_settings'] + tab: ['workspace_settings'], + mode: ['compare'], + items: ['compare'] } // The model-facing schema for the given allowed pages: the `page` enum plus only the @@ -1952,7 +2348,8 @@ const OPEN_PAGE_FIELD_PAGES: Record = { // `trigger_kind` enum is narrowed to the license-available kinds. function buildOpenPageDefSchema( pages: readonly OpenPageName[], - triggerKinds: readonly PageTriggerKind[] + triggerKinds: readonly PageTriggerKind[], + chatEditsTracked: boolean ): z.ZodTypeAny { const full = openPageFullSchema.shape as Record // z.enum() rejects an empty list, and a user with no reachable pages (e.g. an operator @@ -1971,16 +2368,27 @@ function buildOpenPageDefSchema( .enum([...triggerKinds] as [string, ...string[]]) .optional() .describe('Triggers: which trigger kind page to open') - : full[field] + : field === 'items' + ? z + .array(z.string()) + .min(1) + .optional() + .describe(COMPARE_ITEMS_DESCRIPTIONS[chatEditsTracked ? 'tracked' : 'untracked']) + : full[field] } shape.new_tab = full.new_tab return z.object(shape) } const OPEN_PAGE_DESCRIPTION = - 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), or Workspace settings (on a specific tab). Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' + 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its edit drawer opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' -function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs): string { +// Non-arg inputs the URL builder needs: the chat's operating workspace (the compare +// page cannot fall back to its own store default inside a session preview) and the +// live modified-items mask backing the compare page's default preselection. +type OpenPageUrlCtx = { workspaceId: string; chatItems?: readonly string[] } + +export function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs, ctx: OpenPageUrlCtx): string { switch (page) { case 'runs': return buildRunsUrl({ @@ -1996,9 +2404,12 @@ function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs): string { filters: { path: a.path, schedule_path: a.schedule_path, summary: a.summary } }) case 'variables': - return buildVariablesUrl({ path: a.path, owner: a.owner }) + return buildVariablesUrl({ open: a.open, filters: { path: a.path, owner: a.owner } }) case 'resources': - return buildResourcesUrl({ path: a.path, resource_type: a.resource_type, owner: a.owner }) + return buildResourcesUrl({ + open: a.open, + filters: { path: a.path, resource_type: a.resource_type, owner: a.owner } + }) case 'assets': return buildAssetsUrl({ path: a.path }) case 'audit_logs': @@ -2019,6 +2430,15 @@ function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs): string { }) case 'workspace_settings': return buildWorkspaceSettingsUrl({ tab: a.tab }) + case 'compare': + // Explicit `items` wins; otherwise preselect this chat's modified items. An + // empty mask (chat modified nothing) passes no items so the page keeps its + // select-all default instead of preselecting nothing. + return buildCompareUrl({ + workspace_id: ctx.workspaceId, + mode: a.mode, + items: a.items ?? (ctx.chatItems?.length ? ctx.chatItems : undefined) + }) } } @@ -2026,6 +2446,19 @@ function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs): string { // hash target), or "all " when unfiltered. function summarizeOpenPage(url: string, page: OpenPageName): string { const u = new URL(url, 'http://x') + if (page === 'compare') { + // The raw params (workspace_id + a possibly long items list) are noise here — + // summarize the selection instead. + const parts: string[] = [] + const mode = u.searchParams.get('mode') + if (mode) parts.push(`mode=${mode}`) + const items = u.searchParams.get(COMPARE_ITEMS_PARAM) + if (items) { + const n = parseItemsMaskParam(items).size + parts.push(`${n} item${n === 1 ? '' : 's'} preselected`) + } + return parts.length ? parts.join(', ') : 'all pending changes' + } const parts: string[] = [] u.searchParams.forEach((v, k) => parts.push(`${k}=${v}`)) if (u.hash) parts.push(u.hash.slice(1)) @@ -2033,8 +2466,10 @@ function summarizeOpenPage(url: string, page: OpenPageName): string { } export const openPageTool: Tool<{}> = { + // The initial def assumes an untracked chat; setSchema below rebuilds it with the + // caller's real surface before each iteration. def: createToolDef( - buildOpenPageDefSchema(allowedOpenPages(), allowedTriggerKinds()), + buildOpenPageDefSchema(allowedOpenPages(), allowedTriggerKinds(), false), 'open_page', OPEN_PAGE_DESCRIPTION ), @@ -2049,7 +2484,8 @@ export const openPageTool: Tool<{}> = { this.def = createToolDef( buildOpenPageDefSchema( allowedOpenPages(operatingWorkspaceFromHelpers(helpers)), - allowedTriggerKinds() + allowedTriggerKinds(), + (helpers as GlobalToolHelpers | undefined)?.getModifiedItems?.() !== undefined ), 'open_page', OPEN_PAGE_DESCRIPTION @@ -2072,7 +2508,16 @@ export const openPageTool: Tool<{}> = { if (page === 'triggers' && triggerKind && !allowedTriggerKinds().includes(triggerKind)) { return `${TRIGGER_PAGES[triggerKind].label} aren't available on this instance.` } - const url = buildOpenPageUrl(page, parsed) + // Headless callers (ai_evals) have neither helpers.operatingWorkspace nor a + // populated workspaceStore; the chat loop's workspace is still correct there. + const urlWorkspace = workspaceId ?? get(workspaceStore) ?? ctx.workspace + if (!urlWorkspace) { + return 'Error: no workspace is selected, so no page can be opened.' + } + const url = buildOpenPageUrl(page, parsed, { + workspaceId: urlWorkspace, + chatItems: (ctx.helpers as GlobalToolHelpers | undefined)?.getModifiedItems?.() + }) const pageLabel = OPEN_PAGE_LABELS[page] const summary = summarizeOpenPage(url, page) @@ -2111,7 +2556,8 @@ export const globalTools: Tool<{}>[] = [ 'get_instructions', 'Get authoring guidance for scripts, flows, data pipelines, resources, apps, or the datatable SQL SDK (wmill.datatable()) used inside runnables.' ), - fn: async ({ args, toolId, toolCallbacks }) => { + fn: async (ctx) => { + const { args, toolId, toolCallbacks } = ctx const parsed = getInstructionsSchema.parse(args) const label = parsed.subject === 'script' && parsed.language @@ -2122,6 +2568,7 @@ export const globalTools: Tool<{}>[] = [ } }, createSearchHubScriptsTool(false), + searchNpmPackagesTool, searchDocsTool, readDocsPageTool, { @@ -2130,6 +2577,7 @@ export const globalTools: Tool<{}>[] = [ 'askUserQuestion', 'Ask the user a question with proposed answers and wait for their selected or custom answer before continuing.' ), + streamingLabel: 'Asking the user a question...', fn: async ({ args, toolId, toolCallbacks }) => { const parsed = askUserQuestionSchema.parse(args) const userQuestion = { @@ -2139,7 +2587,7 @@ export const globalTools: Tool<{}>[] = [ } toolCallbacks.setToolStatus(toolId, { - content: parsed.question, + content: `Asking user: ${parsed.question}`, userQuestion, isLoading: true }) @@ -2159,7 +2607,7 @@ export const globalTools: Tool<{}>[] = [ if (!selected?.length) { const message = 'Question cancelled by user' toolCallbacks.setToolStatus(toolId, { - content: message, + content: `Asked: ${parsed.question} — cancelled by user`, userQuestion: { ...userQuestion, canceled: true }, isLoading: false, error: message @@ -2173,12 +2621,13 @@ export const globalTools: Tool<{}>[] = [ // ("Yes, immediately") stays unambiguous to the model reading it back. const answerText = selected.length === 1 ? selected[0] : selected.map((c) => `- ${c}`).join('\n') - // The collapsed tool-header is a human glance, not model input, so the picks - // read as a compact comma list there instead of a stacked bullet list. + // The collapsed tool-header is a human glance, not model input, so it carries + // the question plus the picks as a compact comma list (not a bullet list) — + // it is the only place the exchange stays readable in the transcript. const answerSummary = selected.join(', ') toolCallbacks.setToolStatus(toolId, { - content: `User answered question: ${answerSummary}`, + content: `Asked: ${parsed.question} — ${answerSummary}`, userQuestion: { ...userQuestion, selectedChoices: selected @@ -2266,7 +2715,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( listWorkspaceItemsSchema, 'list_workspace_items', - 'List workspace items and drafts. Returns metadata only.' + 'List workspace items and drafts. Returns metadata only, up to limit items per item type per page (default 50); pass page to continue past a full page.' ), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = listWorkspaceItemsSchema.parse(args) @@ -2279,24 +2728,37 @@ export const globalTools: Tool<{}>[] = [ types, workspace, parsed.path_prefix, - Math.min(limit, MAX_LIST_LIMIT) + Math.min(limit, MAX_LIST_LIMIT), + parsed.page ) for (const item of workspaceItems) { byKey.set(getWorkspaceItemKey(item.type, item.path, item.triggerKind), item) } - for (const draft of await listGlobalDrafts(workspace)) { - if (!types.includes(draft.type)) continue - if (parsed.path_prefix && !draft.path.startsWith(parsed.path_prefix)) continue - byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { - ...draft, - value: undefined - }) + // Drafts are not paginated server-side; overlay them on page 1 only, + // capped at `limit` per type, so results stay bounded and later pages + // never repeat a page-1 item as its draft twin. Chat draft counts are + // small — past the cap, a narrower path_prefix still finds any draft + // (it filters before the cap; query filters after). + if ((parsed.page ?? 1) === 1) { + const draftCountByType = new Map() + for (const draft of await listGlobalDrafts(workspace)) { + if (!types.includes(draft.type)) continue + if (parsed.path_prefix && !draft.path.startsWith(parsed.path_prefix)) continue + const count = draftCountByType.get(draft.type) ?? 0 + if (count >= limit) continue + draftCountByType.set(draft.type, count + 1) + byKey.set(getWorkspaceItemKey(draft.type, draft.path, draft.triggerKind), { + ...draft, + value: undefined + }) + } } - const results = Array.from(byKey.values()) - .filter((item) => itemMatches(item, parsed.query)) - .slice(0, limit) + // No cross-type truncation: each type is already capped at `limit` rows by + // its own list call, and slicing the concatenation would silently drop the + // later types' rows while their next page skips past them. + const results = Array.from(byKey.values()).filter((item) => itemMatches(item, parsed.query)) toolCallbacks.setToolStatus(toolId, { content: `Listed ${results.length} workspace item(s)` @@ -2308,7 +2770,7 @@ export const globalTools: Tool<{}>[] = [ def: createToolDef( readWorkspaceItemSchema, 'read_workspace_item', - 'Read one workspace item or draft.' + 'Read one workspace item or draft. Prefers your draft when one exists; pass version: "deployed" to read the deployed state instead.' ), fn: async ({ args, workspace, toolId, toolCallbacks }) => { const parsed = readWorkspaceItemSchema.parse(args) @@ -2317,7 +2779,10 @@ export const globalTools: Tool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: message, error: message }) return JSON.stringify({ success: false, error: message }) } - const draft = await getGlobalDraft(workspace, parsed.type, parsed.path, parsed.trigger_kind) + const draft = + parsed.version === 'deployed' + ? null + : await getGlobalDraft(workspace, parsed.type, parsed.path, parsed.trigger_kind) if (draft) { toolCallbacks.setToolStatus(toolId, { content: `Read draft ${parsed.type} "${parsed.path}"` @@ -2328,7 +2793,13 @@ export const globalTools: Tool<{}>[] = [ toolCallbacks.setToolStatus(toolId, { content: `Reading ${parsed.type} "${parsed.path}"...` }) - const item = await readWorkspaceItem(parsed.type, parsed.path, workspace, parsed.trigger_kind) + const item = await readWorkspaceItem( + parsed.type, + parsed.path, + workspace, + parsed.trigger_kind, + parsed.version === 'deployed' + ) toolCallbacks.setToolStatus(toolId, { content: `Read ${parsed.type} "${parsed.path}"` }) return JSON.stringify(serializeWorkspaceItemForRead(item), null, 2) } @@ -2401,15 +2872,17 @@ export const globalTools: Tool<{}>[] = [ groups: parseOptionalJsonArg(parsed.groups, 'groups'), notes: parseOptionalJsonArg(parsed.notes, 'notes') }) - return writeFlowDraft( + const resolved = await resolveWriteFlowInlineScripts(parsed.path, editable, ctx.workspace) + const result = await writeFlowDraft( { path: parsed.path, summary: parsed.summary, description: parsed.description, - flow: editableFlowToDraftValue(editable) + flow: editableFlowToDraftValue(resolved) }, ctx ) + return appendEmptyInlineScriptWarning(result, resolved) } }, { @@ -2620,6 +3093,26 @@ export const globalTools: Tool<{}>[] = [ return rebaseDraft(parsed, ctx) } }, + { + def: createToolDef( + diffSchema, + 'diff', + "Diff workspace changes. Read-only. Default: drafts vs deployed versions (index without type/path, one item's unified diff with them; file= for one file inside an app). against='parent_workspace': deployed fork vs its parent workspace. search= greps changed lines across all diffs." + ), + showDetails: true, + fn: async (ctx) => { + const parsed = diffSchema.parse(ctx.args) + if (parsed.search !== undefined) { + return diffSearch(parsed, ctx) + } + if (parsed.against === 'parent_workspace') { + return parsed.path !== undefined ? diffForkItem(parsed, ctx) : diffForkIndex(parsed, ctx) + } + return parsed.path !== undefined + ? diffWorkspaceItem(parsed, ctx) + : diffWorkspaceIndex(parsed, ctx) + } + }, { def: createToolDef( deleteWorkspaceItemSchema, @@ -2709,6 +3202,11 @@ export const globalTools: Tool<{}>[] = [ ) } }, + createDbSchemaTool<{}>({ + description: + 'Fetch the schema (tables and columns) of a database resource by its path. Supports postgresql, mysql, ms_sql_server, snowflake and bigquery resources.', + updateEditorCache: false + }), { def: createToolDef( readFlowModuleCodeSchema, @@ -2835,6 +3333,7 @@ export const globalTools: Tool<{}>[] = [ return deleteAppRunnable(parsed, ctx) } }, + ...artifactTools, { def: createToolDef( openPreviewSchema, @@ -2902,10 +3401,117 @@ export const globalTools: Tool<{}>[] = [ return result.aiResult } }, + { + def: createToolDef( + searchDomSchema, + 'search_dom', + 'Search the live rendered HTML of the raw app preview open in this AI session with a regex, returning matching lines with their line numbers. Use it to check what actually rendered (verify an edit landed, diagnose a blank/empty view). Scope to an element with `selector`, or omit it for the whole page. The DOM is read live, so it reflects the current state.' + ), + showDetails: true, + fn: async (ctx) => { + const parsed = searchDomSchema.parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: 'Searching app DOM...' }) + const result = await getSessionDom( + { + mode: 'search', + appPath: parsed.app_path, + selector: parsed.selector, + pattern: parsed.pattern, + ignoreCase: parsed.ignore_case + }, + sessionIdFromCtx(ctx) + ) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: result.uiMessage, + result: result.toolResult + }) + return result.aiResult + } + }, + { + def: createToolDef( + readDomSchema, + 'read_dom', + 'Read a bounded window of the live rendered HTML of the raw app preview open in this AI session, pretty-printed and line-numbered. Scope to an element with `selector`, or omit it for the whole page. Use search_dom first to locate content, then read_dom to see a specific region. The DOM is read live.' + ), + showDetails: true, + fn: async (ctx) => { + const parsed = readDomSchema.parse(ctx.args) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: 'Reading app DOM...' }) + const result = await getSessionDom( + { + mode: 'read', + appPath: parsed.app_path, + selector: parsed.selector, + startLine: parsed.start_line, + endLine: parsed.end_line + }, + sessionIdFromCtx(ctx) + ) + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: result.uiMessage, + result: result.toolResult + }) + return result.aiResult + } + }, + { + def: createToolDef( + takeScreenshotSchema, + 'take_screenshot', + // Keep this short: every global session iteration re-sends it. How to read + // the result belongs on the result, where only a real capture pays for it. + 'Capture a screenshot of the raw app preview currently open in this AI session and attach it as an image so you can see the rendered UI. Use it when the user raises how the app looks, whether reporting a problem or asking for the design improved, rather than to check your own edits. The image is attached in the following message. Requires the raw app preview open (open_preview kind="raw_app").' + ), + showDetails: true, + fn: async (ctx) => { + // A known text-only model would reject the follow-up image message and fail + // the turn, so refuse before capturing rather than buffer an image it can + // never read. The model is re-read here because it can change between turns. + const model = tryGetCurrentModel() + if (model && !modelSupportsVision(model.provider, model.model)) { + const cannotSee = `${model.model} cannot read images, so a screenshot would be discarded. Ask the user to describe what looks wrong, or to switch to a model that supports images.` + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: `${model.model} cannot read images`, + error: cannotSee + }) + return cannotSee + } + ctx.toolCallbacks.setToolStatus(ctx.toolId, { content: 'Capturing screenshot...' }) + const result = await getSessionScreenshot(sessionIdFromCtx(ctx)) + if (!result.dataUrl) { + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: result.uiMessage ?? 'Screenshot unavailable', + error: result.error + }) + return result.error ?? 'Could not capture the app preview.' + } + // Normalize (downscale + png/jpeg) so history/context never carry a full-res blob; + // buffered here and flushed as a follow-up user image message once the tool batch + // completes (see appendPendingToolImages). + const image = await normalizeImageDataUrl(result.dataUrl) + ctx.toolCallbacks.attachToolImage?.(ctx.toolId, image) + // The card shows the same copy the model gets; sharing the exact data URL + // lets the history's blob store persist one copy for both. + ctx.toolCallbacks.setToolStatus(ctx.toolId, { + content: 'Screenshot captured', + imageUrl: image.dataUrl + }) + return ( + 'Screenshot captured; the image is attached in the following message.\n\n' + + 'It is rebuilt from the DOM rather than captured from the screen, so it can differ from what the user sees, and it differs by browser. Treat what you see as real and fix it. Before dismissing anything as a capture artifact, read the source for that element and name the specific cause; if you cannot, it is a real bug. If you are still unsure, say what looks wrong and ask the user to screenshot it themselves and drag the image into the chat rather than guessing.' + ) + } + }, // Workspace-scoped datatable tools (unrestricted: no whitelist, no creation policy) ...getDatatableTools(), + // Workspace DuckLake readiness (storage prerequisite check for pipelines) + ...getDucklakeTools(), // Read-only tools over files the user attached to the conversation - ...fileTools + ...fileTools, + // Search + call access to the backend API endpoint catalog, for operations + // no dedicated tool covers + ...apiCatalogTools ] // Tools that only make sense inside an AI session (they drive the session's @@ -2916,7 +3522,14 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ 'get_preview_status', 'close_page', 'get_app_runtime_logs', - 'list_app_runs' + 'list_app_runs', + 'search_dom', + 'read_dom', + 'take_screenshot', + 'create_artifact', + 'update_artifact', + 'list_artifacts', + 'read_artifact' ]) /** @@ -2925,9 +3538,16 @@ export const SESSION_PREVIEW_TOOL_NAMES = new Set([ * global side-panel chat. */ export function globalToolsFor({ sessionPreview }: { sessionPreview: boolean }): Tool<{}>[] { - return sessionPreview + const tools = sessionPreview ? globalTools : globalTools.filter((t) => !SESSION_PREVIEW_TOOL_NAMES.has(t.def.function.name)) + // DOM capture re-renders the app through the engine's SVG-image path, which is + // only faithful on Blink — Gecko/WebKit shift text spacing and wrapping (font + // fallback, sub-pixel rounding). Elsewhere the tool is withheld entirely and + // the system prompt tells the agent to ask the user for a screenshot instead. + return isChromiumBrowser() + ? tools + : tools.filter((t) => t.def.function.name !== 'take_screenshot') } type WriteDraftCtx = { @@ -2938,6 +3558,9 @@ type WriteDraftCtx = { // reloads the preview of the session that issued the deploy — not the // UI-active one. Undefined for the global side-panel chat. sessionId?: string + // Present when the ctx is the raw tool `fn` context — session chats carry + // their id here (see SessionToolHelpers / sessionIdFromCtx). + helpers?: unknown } // Sessions are the only context where `open_preview` makes sense — the global @@ -2964,6 +3587,14 @@ export type GlobalToolHelpers = SessionToolHelpers & { // (possibly forked) workspace while $workspaceStore stays on the navigation workspace, // so permission gating (open_page) must read this, not the global store. operatingWorkspace?: string + // Wired only for session chats (see AIChatManager): the artifact tools are session-gated. + artifacts?: SessionArtifactsStore + getChatId?: () => string | undefined + // Live snapshot of the items this chat modified (`kind:path` mask keys, see + // modifiedItemsMask.ts); undefined when the chat doesn't track them (the global + // side-panel chat). Backs open_page's compare-page default preselection. + getModifiedItems?: () => string[] | undefined + openArtifact?: (artifactId: string, name: string) => void } function sessionIdFromCtx(ctx: { helpers?: unknown }): string | undefined { @@ -2989,7 +3620,7 @@ export type OpenPreviewHandler = (req: { sessionId: string | undefined kind: 'script' | 'flow' | 'raw_app' | 'pipeline' path: string -}) => string +}) => string | Promise let openPreviewHandler: OpenPreviewHandler | undefined @@ -2997,14 +3628,17 @@ export function setOpenPreviewHandler(handler: OpenPreviewHandler | undefined): openPreviewHandler = handler } -function openSessionPreview( +async function openSessionPreview( args: { kind: 'script' | 'flow' | 'raw_app' | 'pipeline'; path: string }, sessionId: string | undefined -) { +): Promise { if (!openPreviewHandler) { return 'Error: open_preview is only available inside an AI session. Tell the user to switch to a session to view the preview, or describe the item textually.' } - return openPreviewHandler({ ...args, sessionId }) + // open_preview only exists in sessions, so no sessionId check is needed here. + // For a pipeline the handler awaits the editor's tool registration, so the + // model's next build_pipeline_node call can't race the async canvas mount. + return await openPreviewHandler({ ...args, sessionId }) } // Opens a workspace *page* (Runs, Schedules, …) as a page tab in the session's @@ -3135,6 +3769,54 @@ function getSessionAppRuns( return Promise.resolve(listAppRunsHandler({ sessionId, limit })) } +export type GetDomHandler = (req: { + sessionId: string | undefined + query: RawAppDomQuery +}) => Promise + +let getDomHandler: GetDomHandler | undefined + +export function setGetDomHandler(handler: GetDomHandler | undefined): void { + getDomHandler = handler +} + +function getSessionDom( + query: RawAppDomQuery, + sessionId: string | undefined +): Promise { + if (!getDomHandler) { + return Promise.resolve({ + aiResult: + 'Error: search_dom and read_dom are only available inside an AI session. Tell the user the rendered DOM can only be read from a session preview, or switch to a session and open the raw app preview.', + uiMessage: 'DOM unavailable', + toolResult: 'DOM unavailable' + }) + } + return getDomHandler({ sessionId, query }) +} + +export type SessionScreenshotResult = { dataUrl?: string; error?: string; uiMessage?: string } +export type ScreenshotHandler = (req: { + sessionId: string | undefined +}) => Promise + +let screenshotHandler: ScreenshotHandler | undefined + +export function setScreenshotHandler(handler: ScreenshotHandler | undefined): void { + screenshotHandler = handler +} + +function getSessionScreenshot(sessionId: string | undefined): Promise { + if (!screenshotHandler) { + return Promise.resolve({ + error: + 'Error: take_screenshot is only available inside an AI session with a raw app preview open. Ask the user to open the raw app preview (open_preview kind="raw_app"), then try again.', + uiMessage: 'Screenshot unavailable' + }) + } + return screenshotHandler({ sessionId }) +} + // Registered by the session runtime to reload the open preview after a chat // deploy. Undefined outside a session. export type DeployedInSessionHandler = (req: { @@ -3309,6 +3991,34 @@ function draftWriteFailure(result: DraftPersistResult, ctx: WriteDraftCtx): stri return undefined } +// Item kinds a session preview can host, keyed by the draft item kind a write +// resolves to. Kinds absent here (resources, variables, triggers, legacy `app`) +// have no preview panel, so no card is offered for them. +const PREVIEW_CARD_KIND_BY_ITEM_KIND: Partial< + Record +> = { + script: 'script', + flow: 'flow', + raw_app: 'raw_app' +} + +// Offer a preview card for a write that landed a previewable item. Session chats +// only: the card opens the item in the side panel, which the global side-panel +// chat has no equivalent of. `path` is the item's display path (what +// `open_preview` takes), not its synthetic draft storage key. +function maybeAttachPreviewCard( + ctx: WriteDraftCtx, + itemKind: DraftPersistResult['itemKind'], + path: string +): void { + // Write tools pass the raw tool ctx, whose session id lives in `helpers` — + // `ctx.sessionId` is only set by callers that thread it explicitly. + if (!ctx.sessionId && !sessionIdFromCtx(ctx)) return + const kind = PREVIEW_CARD_KIND_BY_ITEM_KIND[itemKind] + if (!kind) return + ctx.toolCallbacks.setToolStatus(ctx.toolId, { previewCard: { kind, path } }) +} + // App write tools build varied success messages but share the same conflict / // save-failure handling; `onSaved` supplies the per-tool status + message. function finishAppDraftWrite( @@ -3319,6 +4029,7 @@ function finishAppDraftWrite( const failure = draftWriteFailure(result, ctx) if (failure) return failure ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) + maybeAttachPreviewCard(ctx, result.itemKind, result.item.path) const { content, message } = onSaved() ctx.toolCallbacks.setToolStatus(ctx.toolId, { content, result: 'Saved as draft' }) return JSON.stringify({ success: true, message }, null, 2) @@ -3332,6 +4043,7 @@ function finishDraftWrite( const failure = draftWriteFailure(result, ctx) if (failure) return failure ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) + maybeAttachPreviewCard(ctx, result.itemKind, result.item.path) const stored = result.item const verb = existed ? 'Updated' : 'Created' // Don't echo the flow value back: the model just sent it in the write call, @@ -3519,7 +4231,17 @@ function triggerWriteSpec(kind: TriggerKind): WriteSpec service.exists({ workspace, path }), fetchDeployed: async (workspace, path) => (await service.get({ workspace, path })) as TriggerDraftConfig, - buildDraft: (base, config, path) => mergeDraftConfig(base, config, path) + buildDraft: (base, config, path) => { + const draft = mergeDraftConfig(base, config, path) + if (kind === 'email') { + // workspaced_local_part maps to a NOT NULL column but is optional in the + // tool schema. Default on the merged draft (not the incoming config) so an + // omitted field keeps the existing trigger's value instead of resetting it. + const email = draft as TriggerDraftConfig & { workspaced_local_part?: boolean } + email.workspaced_local_part = email.workspaced_local_part ?? false + } + return draft + } } } @@ -3618,6 +4340,46 @@ async function loadFlowDraftValue( } } +/** + * `write_flow` accepts `inline_script.` placeholders so the model can + * overwrite a flow without re-sending (or even reading) unchanged rawscript + * bodies. Resolve them against the current draft/deployed flow: an id with a + * stored body keeps it, a new module's own-id placeholder becomes an empty body + * (to fill via set_flow_module_code), and anything else rejects the write. + */ +async function resolveWriteFlowInlineScripts( + path: string, + editable: EditableFlowJson, + workspace: string +): Promise { + const specials = [editable.preprocessor_module, editable.failure_module].filter( + (module): module is FlowModule => module != null + ) + const hasPlaceholders = + findUnresolvedInlineScriptRefs(editable.modules).length > 0 || + findUnresolvedInlineScriptRefs(specials).length > 0 + if (!hasPlaceholders) { + return editable + } + + const session = createInlineScriptSession() + if ( + (await getGlobalDraft(workspace, 'flow', path)) || + (await FlowService.existsFlowByPath({ workspace, path })) + ) { + const base = await loadFlowDraftValue(path, workspace) + buildEditableFlowJson(flowDraftAsEditableInput(base.flow), session) + } + const resolved: EditableFlowJson = { + ...editable, + modules: session.restoreInlineScriptReferences(editable.modules), + preprocessor_module: restoreSpecialRawscriptModule(editable.preprocessor_module, session), + failure_module: restoreSpecialRawscriptModule(editable.failure_module, session) + } + finalizeUnresolvedInlineScripts(resolved) + return resolved +} + async function patchFlowJson( args: { path: string; old_string: string; new_string: string; replace_all: boolean }, ctx: WriteDraftCtx @@ -3650,8 +4412,9 @@ async function patchFlowJson( const patchedEditable = validateEditableFlowJson(parsedValue) const newFlowValue = applyEditableFlowJsonToFlow(base.flow.value, patchedEditable, session) + finalizeUnresolvedInlineScripts(newFlowValue) - return writeFlowDraft( + const result = await writeFlowDraft( { path, summary: base.summary, @@ -3664,6 +4427,15 @@ async function patchFlowJson( }, ctx ) + // Warn from the restored value, not patchedEditable — the compact view holds + // placeholders for every rawscript, so only post-restore content shows which + // modules still need bodies filled via set_flow_module_code. + return appendEmptyInlineScriptWarning(result, { + ...patchedEditable, + modules: newFlowValue.modules, + preprocessor_module: newFlowValue.preprocessor_module ?? null, + failure_module: newFlowValue.failure_module ?? null + }) } async function readFlowModuleCode( @@ -4387,9 +5159,11 @@ const triggerLabels: Record = { nats: 'NATS trigger', postgres: 'Postgres trigger', mqtt: 'MQTT trigger', + amqp: 'AMQP trigger', sqs: 'SQS trigger', gcp: 'GCP Pub/Sub trigger', - azure: 'Azure Event Grid trigger' + azure: 'Azure Event Grid trigger', + email: 'Email trigger' } function createOpenScheduleAction(path: string, targetKind: 'script' | 'flow'): ToolDisplayAction { @@ -4747,6 +5521,776 @@ async function rebaseAppDraft(path: string, ctx: WriteDraftCtx): Promise ) } +const MAX_DIFF_PATCH_CHARS = 50_000 + +function windowPatchBody(patch: string, offset: number, limit: number): string { + return windowPatch(patch, offset, limit, MAX_DIFF_PATCH_CHARS) +} + +const DIFF_READ_DEFAULT_LINES = 500 +const DIFF_INDEX_DEFAULT_ITEMS = 50 +const DIFF_INDEX_MAX_ITEMS = 100 + +// Read-only draft-vs-deployed diff for one draftable item, served from the +// workspace diff snapshot (fetched once, shared with the index), with a direct +// computation fallback when no draft row is listed. +async function diffWorkspaceItem( + args: { + type?: WorkspaceItemType + path?: string + trigger_kind?: TriggerKind + file?: string + offset?: number + limit?: number + }, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const { type, path, trigger_kind: triggerKind } = args + if (!type || !path) { + throw new Error('type is required when path is provided.') + } + const itemKind = itemKindFor(type, triggerKind) + if (!itemKind) { + throw new Error('trigger_kind is required when type is trigger.') + } + toolCallbacks.setToolStatus(toolId, { + content: `Comparing draft vs deployed for "${path}"...` + }) + + // Address the draft by its storage path (a draft_only item lives at a + // synthetic `u/{user}/draft_{uuid}` key) and flush any parked editor autosave + // first so the server overlay reflects the latest edit — the same resolution + // the deploy path uses. Unlike deploy: a flush conflict/failure doesn't abort + // a read-only diff (surfaced as a caveat), and the auto-save toggle is + // honored — with auto-save off, a read-only tool must not persist edits the + // user chose to keep local. The just-flushed row must not be served from the + // throttled listing cache, so expire it. + // The chat `app` type spans two draft kinds: raw apps AND classic apps — + // the classic editor parks its cell under `app`, so both keys must be + // flushed and probed or classic edits silently go stale. Each kind resolves + // its own storage path (live-editor mapping), and the listing resolves a + // friendly/renamed path to the row that owns it — a renamed classic app's + // cell lives at its ORIGINAL storage path, which only the listing knows. + const draftKinds: UserDraftItemKind[] = type === 'app' ? ['raw_app', 'app'] : [itemKind] + const flushQueries = draftKinds.map((kind) => ({ + workspace, + itemKind: kind, + path: resolveGlobalDraftStoragePathByKind(workspace, kind, path) + })) + const listedTarget = await resolveWorkspaceDiffTarget(workspace, draftKinds, path) + if ( + listedTarget && + !flushQueries.some( + (q) => q.itemKind === listedTarget.kind && q.path === listedTarget.storagePath + ) + ) { + flushQueries.push({ workspace, itemKind: listedTarget.kind, path: listedTarget.storagePath }) + } + const storagePath = listedTarget?.storagePath ?? flushQueries[0].path + for (const query of flushQueries) { + await UserDraftDbSyncer.flush(query, { honorAutosaveToggle: true }) + } + expireWorkspaceDiffList(workspace) + const hasConflict = flushQueries.some((query) => UserDraftDbSyncer.getConflict(query).conflict) + + // When the latest edits never reached the server (auto-save off, the save + // failed, or it conflicted with a newer server version), the persisted + // state is stale: diff against the in-memory editor value instead — + // read-only, nothing gets persisted — and bypass the snapshot cache, + // which must only ever hold persisted state. + let flushSkipped = false + let localValue: unknown + let localKind: UserDraftItemKind = itemKind + let localPath = storagePath + for (const query of flushQueries) { + const skipped = + UserDraftDbSyncer.hasUnsavedDisabledChanges(query) || + UserDraftDbSyncer.getState(query).state === 'failed' || + UserDraftDbSyncer.getConflict(query).conflict + if (!skipped) continue + flushSkipped = true + const cell = readLocalDraftCellByKind(workspace, query.itemKind, query.path) + if (cell !== undefined) { + localValue = cell + localKind = query.itemKind + localPath = query.path + break + } + } + + let flushCaveat = hasConflict + ? localValue !== undefined + ? "Warning: the draft conflicts with a newer server version; this diff shows YOUR local editor value, not the server's. Resolve the conflict in the editor before deploying.\n\n" + : 'Warning: the draft has a conflicting newer version on the server; this diff shows the persisted draft, which may not include the latest editor edits.\n\n' + : '' + let patch: string + let noDeployed: boolean + let files: Record | undefined + let valueUncomparable = false + if (localValue !== undefined) { + let deployedSide: unknown + try { + const values = await getDraftDiffValues(localKind, localPath, workspace) + noDeployed = values.noDeployed + deployedSide = noDeployed ? undefined : values.deployed + } catch (e) { + if ((e as { status?: number } | null | undefined)?.status !== 404) throw e + // Editor-only draft that was never persisted at all. + noDeployed = true + } + let beforeSide = deployedSide + let afterSide = canonicalDraftSideValue(localKind, localValue) + // App sides carry `path` (staged renames diff); mirror it onto the local + // canonical value, which only knows a draft_path. + if (localKind === 'app' || localKind === 'raw_app') { + const deployedPath = (deployedSide as { path?: string } | undefined)?.path ?? localPath + const stagedPath = (localValue as { draft_path?: string } | null)?.draft_path + afterSide = { ...(afterSide as Record), path: stagedPath ?? deployedPath } + if (deployedSide !== undefined) { + beforeSide = { ...(deployedSide as Record), path: deployedPath } + } + } + if (itemKind === 'variable') { + ;({ + before: beforeSide, + after: afterSide, + valueUncomparable + } = maskVariableDiffSides(beforeSide, afterSide)) + flushCaveat += valueUncomparable ? SECRET_UNCOMPARABLE_NOTE : VARIABLE_MASKED_NOTE + } + const parts = computeDiffParts(beforeSide, afterSide, 'deployed', 'draft') + patch = parts.patch + files = parts.files + flushCaveat += + 'Note: this diff includes unsaved editor changes that are NOT saved to the server draft yet (auto-save is off or the last save failed).\n\n' + } else if (flushSkipped) { + throw new Error( + `The latest editor changes for ${type} "${path}" could not be saved and are not readable; retry once the editor saves.` + ) + } else { + const entry = await readWorkspaceDiffEntry(workspace, itemKind, storagePath) + if (entry) { + if (entry.status === 'error') { + throw new Error(`Could not diff ${type} "${path}": ${entry.errorMessage}`) + } + patch = entry.patch ?? '' + noDeployed = entry.noDeployed === true + files = entry.files + valueUncomparable = entry.valueUncomparable === true + if (entry.valueMasked) { + flushCaveat += valueUncomparable ? SECRET_UNCOMPARABLE_NOTE : VARIABLE_MASKED_NOTE + } + } else { + // Not in the draft listing — either no draft at all (deployed is current), + // nothing at the path, or a listing/overlay disagreement; ask the overlay. + let values: Awaited> + try { + values = await getDraftDiffValues(itemKind, storagePath, workspace) + } catch (e) { + if ((e as { status?: number } | null | undefined)?.status === 404) { + throw new Error( + `No ${type} found at "${path}" — it has neither a deployed version nor a draft.` + ) + } + throw e + } + const { deployed, draft, hasDraft, noDeployed: fetchedNoDeployed } = values + if (!fetchedNoDeployed && !hasDraft) { + const message = `No draft exists for ${type} "${path}" — the deployed version is current.` + toolCallbacks.setToolStatus(toolId, { content: message }) + return message + } + // A never-deployed item diffs against nothing: the whole draft reads as added. + noDeployed = fetchedNoDeployed + let beforeSide: unknown = noDeployed ? undefined : deployed + let afterSide: unknown = draft + if (itemKind === 'variable') { + ;({ + before: beforeSide, + after: afterSide, + valueUncomparable + } = maskVariableDiffSides(beforeSide, afterSide)) + flushCaveat += valueUncomparable ? SECRET_UNCOMPARABLE_NOTE : VARIABLE_MASKED_NOTE + } + patch = draftDeployedPatch(beforeSide, afterSide) + } + } + + const changedFileCount = files ? Object.keys(files).length : 0 + if (!patch && changedFileCount === 0) { + // A secret's sides are masked on both ends — an empty patch cannot prove + // the value is unchanged. + const message = valueUncomparable + ? `No visible changes for ${type} "${path}" — but a secret's value cannot be compared and may have been updated in the draft.` + : `Draft matches the deployed version of ${type} "${path}" — no changes.` + toolCallbacks.setToolStatus(toolId, { content: message, result: message }) + return flushCaveat + message + } + + const header = noDeployed + ? `${type} "${path}" has no deployed version yet — the entire draft is new.\n\n` + : `Draft changes vs deployed for ${type} "${path}":\n\n` + if (args.file !== undefined && !files) { + throw new Error( + `file only applies to multi-file apps; ${type} "${path}" diffs as a single document — call again without file.` + ) + } + const body = files + ? renderEntryFiles(files, patch, args) + : windowPatchBody(patch, args.offset ?? 0, args.limit ?? DIFF_READ_DEFAULT_LINES) + const result = flushCaveat + header + body + toolCallbacks.setToolStatus(toolId, { + content: `Draft vs deployed diff for "${path}"`, + result + }) + return result +} + +// Body of an item read for a multi-file app: one file's patch when `file` is +// given, otherwise the per-file summary plus config changes. +function renderEntryFiles( + files: Record, + configPatch: string, + args: { file?: string; offset?: number; limit?: number } +): string { + if (args.file !== undefined) { + // App files are keyed with a leading slash ("/App.tsx") — accept the + // slash-less spelling and a unique basename too. + const names = Object.keys(files) + const requested = args.file + const resolved = + names.find((n) => n === requested) ?? + names.find((n) => n === `/${requested}`) ?? + (names.filter((n) => n.endsWith(`/${requested.replace(/^\//, '')}`)).length === 1 + ? names.find((n) => n.endsWith(`/${requested.replace(/^\//, '')}`)) + : undefined) + if (resolved === undefined) { + const changed = names.join(', ') || '(none)' + throw new Error(`No changes in file "${requested}". Changed files: ${changed}.`) + } + const fileDiff = files[resolved] + if (fileDiff.patch === '') { + // Empty file added/deleted: the presence change IS the whole diff. + return `File "${resolved}" was ${fileDiff.status} with empty content.` + } + return windowPatchBody(fileDiff.patch, args.offset ?? 0, args.limit ?? DIFF_READ_DEFAULT_LINES) + } + const sections: string[] = [] + const fileLines = Object.entries(files).map( + ([name, fileDiff]) => + `- ${name} — ${fileDiff.status}${fileDiff.status === 'deleted' ? '' : fileDiff.lineCount === 0 ? ' (empty file)' : ` (${fileDiff.lineCount} diff lines)`}` + ) + sections.push( + fileLines.length > 0 + ? `${fileLines.length} file(s) changed:\n${fileLines.join('\n')}\nRead one with file="".` + : 'No file contents changed.' + ) + if (configPatch) { + sections.push( + 'Config changes:\n' + + windowPatchBody(configPatch, args.offset ?? 0, args.limit ?? DIFF_READ_DEFAULT_LINES) + ) + } + return sections.join('\n\n') +} + +// Indented per-file child rows under a multi-file app's index line. +const DIFF_INDEX_MAX_FILE_CHILDREN = 20 +function fileChildrenLines(e: { files?: Record; patch?: string }): string[] { + if (!e.files) return [] + const names = Object.keys(e.files) + if (names.length === 0 && !e.patch) return [] + const lines = names.slice(0, DIFF_INDEX_MAX_FILE_CHILDREN).map((name) => { + const fileDiff = e.files![name] + return ` · ${name} — ${fileDiff.status}${ + fileDiff.status === 'deleted' + ? '' + : fileDiff.lineCount === 0 + ? ' (empty file)' + : ` (${fileDiff.lineCount} lines)` + }` + }) + if (names.length > DIFF_INDEX_MAX_FILE_CHILDREN) { + lines.push(` · … ${names.length - DIFF_INDEX_MAX_FILE_CHILDREN} more files`) + } + if (e.patch) { + lines.push(` · (config) — modified (${e.patch.split('\n').length} lines)`) + } + return lines +} + +function formatDiffIndexEntry(e: WorkspaceDiffEntryView): string { + const label = e.type === 'trigger' ? `${e.triggerKind} trigger` : (e.type ?? e.kind) + const name = `${label} "${e.path}"` + switch (e.status) { + case 'new': + return `- ${name} — new, never deployed (${e.patchLineCount} lines)` + case 'modified': + return e.valueUncomparable + ? `- ${name} — modified (${e.patchLineCount} diff lines; secret value may also differ)` + : `- ${name} — modified (${e.patchLineCount} diff lines)` + case 'unchanged': + return e.valueUncomparable + ? `- ${name} — no visible changes (secret value cannot be compared; may differ)` + : `- ${name} — draft matches deployed` + case 'pending': + return `- ${name} — draft present (diff not computed yet; read it with type+path)` + case 'error': + return `- ${name} — diff failed: ${e.errorMessage}` + case 'not_diffable': + return `- ${e.kind} draft "${e.path}" — not addressable in this chat` + } +} + +// Workspace index: every draft the current user has, with its change status +// from the materialized snapshot. +async function diffWorkspaceIndex( + args: { types?: WorkspaceItemType[]; path_prefix?: string; limit?: number }, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + toolCallbacks.setToolStatus(toolId, { content: 'Computing workspace draft diff...' }) + // Parked editor autosaves may not have a server row yet (a brand-new draft + // only appears in the listing after its first flush). + const { unflushedPaths } = await flushGlobalDraftSaves(workspace) + expireWorkspaceDiffList(workspace) + const index = await getWorkspaceDiffIndex(workspace) + let entries = index.entries + if (args.types?.length) { + entries = entries.filter((e) => e.type !== undefined && args.types!.includes(e.type)) + } + if (args.path_prefix) { + entries = entries.filter( + (e) => e.path.startsWith(args.path_prefix!) || e.storagePath.startsWith(args.path_prefix!) + ) + } + const total = entries.length + const shown = entries.slice( + 0, + Math.min(args.limit ?? DIFF_INDEX_DEFAULT_ITEMS, DIFF_INDEX_MAX_ITEMS) + ) + const lines = shown.flatMap((e) => [formatDiffIndexEntry(e), ...fileChildrenLines(e)]) + const notes: string[] = [] + if (total > shown.length) { + notes.push( + `Showing ${shown.length} of ${total} drafts — narrow with types/path_prefix or raise limit.` + ) + } + if (index.otherUsersDraftCount > 0) { + notes.push( + `${index.otherUsersDraftCount} draft(s) by other users exist in this workspace (not shown — drafts are per-user).` + ) + } + if (unflushedPaths.length > 0) { + notes.push( + `Warning: unsaved editor changes on ${unflushedPaths.join(', ')} are NOT reflected here (auto-save off, a save failed, or a conflict is unresolved). Read the item with type+path to include them.` + ) + } + const filtersActive = (args.types?.length ?? 0) > 0 || !!args.path_prefix + const summaryLine = + total === 0 + ? filtersActive && index.entries.length > 0 + ? `No drafts match your filters (${index.entries.length} draft(s) exist in the workspace).` + : 'No drafts in this workspace — nothing differs from the deployed state.' + : `${total} draft(s) vs deployed:` + const result = [summaryLine, ...lines, ...notes].join('\n') + toolCallbacks.setToolStatus(toolId, { + content: `Workspace diff: ${total} draft(s)`, + result + }) + return result +} + +function forkComparisonUnavailableMessage(parent: string): string { + return `The comparison with parent workspace "${parent}" is unavailable for this fork (created before comparison tracking existed).` +} + +function forkParentOrThrow(workspace: string): string { + const parent = getForkParentWorkspaceId(workspace) + if (!parent) { + throw new Error( + `Workspace "${workspace}" is not a fork — it has no parent workspace to compare against. Use diff without 'against' to compare drafts vs deployed versions.` + ) + } + return parent +} + +function forkEntryLabel(e: ForkDiffEntryView): string { + const label = e.type === 'trigger' ? `${e.triggerKind} trigger` : (e.type ?? e.kind) + return `${label} "${e.path}"` +} + +function formatForkIndexEntry(e: ForkDiffEntryView): string { + const name = forkEntryLabel(e) + const draftFlag = e.hasLocalDraft ? ' [+ local draft]' : '' + const aheadBehind = [ + e.ahead > 0 ? `ahead ${e.ahead}` : undefined, + e.behind > 0 ? `behind ${e.behind}` : undefined + ] + .filter(Boolean) + .join(', ') + switch (e.status) { + case 'only_in_fork': + return `- ${name} — only in fork (${e.patchLineCount} lines)${draftFlag}` + case 'deleted_in_fork': + return `- ${name} — deleted in fork, still in parent${draftFlag}` + case 'modified': + return `- ${name} — differs (${aheadBehind}; ${e.patchLineCount} diff lines)${draftFlag}` + case 'unchanged': + // Folder display_name lives only in the DB (no API surface exposes + // it), so an identical projection cannot prove folder parity. + if (e.kind === 'folder') { + const suffix = aheadBehind ? ` (${aheadBehind})` : '' + return `- ${name} — no comparable differences; the folder display name is not exposed by the API and may differ${suffix}${draftFlag}` + } + return e.valueMasked + ? `- ${name} — value never shown in chat; may differ (${aheadBehind})${draftFlag}` + : `- ${name} — content matches parent (version history differs: ${aheadBehind})${draftFlag}` + case 'pending': + return e.type !== undefined + ? `- ${name} — differs (${aheadBehind}; diff not computed yet, read it with type+path)${draftFlag}` + : `- ${name} — differs (${aheadBehind}; diff not computed yet, read it by path alone)${draftFlag}` + case 'error': + return `- ${name} — diff failed: ${e.errorMessage}${draftFlag}` + } +} + +// Comparison kinds a chat (type, trigger_kind) pair addresses. +function forkKindsFor(type: WorkspaceItemType, triggerKind?: TriggerKind): string[] { + switch (type) { + case 'app': + return ['app', 'raw_app'] + case 'trigger': + return triggerKind ? [`${triggerKind}_trigger`] : [] + default: + return [type] + } +} + +// Fork index: deployed fork vs deployed parent, same tally as the fork banner. +async function diffForkIndex( + args: { types?: WorkspaceItemType[]; path_prefix?: string; limit?: number }, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const parent = forkParentOrThrow(workspace) + toolCallbacks.setToolStatus(toolId, { + content: `Comparing fork with parent workspace "${parent}"...` + }) + const index = await getForkDiffIndex(workspace, parent) + if (index.skippedComparison) { + const message = forkComparisonUnavailableMessage(parent) + toolCallbacks.setToolStatus(toolId, { content: message }) + return message + } + let entries = index.entries + if (args.types?.length) { + entries = entries.filter((e) => e.type !== undefined && args.types!.includes(e.type)) + } + if (args.path_prefix) { + entries = entries.filter((e) => e.path.startsWith(args.path_prefix!)) + } + const total = entries.length + const shown = entries.slice( + 0, + Math.min(args.limit ?? DIFF_INDEX_DEFAULT_ITEMS, DIFF_INDEX_MAX_ITEMS) + ) + const lines = shown.flatMap((e) => [formatForkIndexEntry(e), ...fileChildrenLines(e)]) + const notes: string[] = [] + if (total > shown.length) { + notes.push( + `Showing ${shown.length} of ${total} items — narrow with types/path_prefix or raise limit.` + ) + } + if (shown.some((e) => e.hasLocalDraft)) { + notes.push( + '[+ local draft]: you also have an undeployed draft there — not part of this deployed-vs-deployed comparison; use diff without against to see it.' + ) + } + const hasHidden = index.hiddenAheadCount > 0 || index.hiddenBehindCount > 0 + if (hasHidden) { + notes.push( + `Hidden items you lack permission to view also differ: ${index.hiddenAheadCount} ahead, ${index.hiddenBehindCount} behind (a conflicted item counts in both).` + ) + } + const forkFiltersActive = (args.types?.length ?? 0) > 0 || !!args.path_prefix + const summaryLine = + total === 0 + ? forkFiltersActive && index.entries.length > 0 + ? `No differing items match your filters (${index.entries.length} differing item(s) exist).` + : hasHidden + ? `No differences visible to you between this fork and its parent "${parent}" — but hidden items differ (see below).` + : `This fork matches its parent workspace "${parent}" — no differences.` + : `${total} item(s) differ between this fork and its parent "${parent}":` + const result = [summaryLine, ...lines, ...notes].join('\n') + toolCallbacks.setToolStatus(toolId, { + content: `Fork vs parent: ${total} differing item(s)`, + result + }) + return result +} + +// One item's fork-vs-parent unified diff (deployed sides only). +async function diffForkItem( + args: { + type?: WorkspaceItemType + path?: string + trigger_kind?: TriggerKind + file?: string + offset?: number + limit?: number + }, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const { type, path, trigger_kind: triggerKind } = args + if (!path) { + throw new Error('path is required.') + } + const parent = forkParentOrThrow(workspace) + // No type = path-only wildcard: comparison kinds outside the chat type enum + // (folder, resource_type, …) are only reachable this way. + const kinds = type ? forkKindsFor(type, triggerKind) : [] + if (type === 'trigger' && kinds.length === 0) { + throw new Error('trigger_kind is required when type is trigger.') + } + toolCallbacks.setToolStatus(toolId, { + content: `Comparing fork vs parent for "${path}"...` + }) + if ((await getForkComparisonStatus(workspace, parent)).skippedComparison) { + const message = forkComparisonUnavailableMessage(parent) + toolCallbacks.setToolStatus(toolId, { content: message }) + return message + } + const entries = await readForkDiffEntries(workspace, parent, kinds, path) + if (entries.length === 0) { + const message = `${type ?? 'item'} "${path}" does not differ between this fork and its parent "${parent}" (or does not exist in either).` + toolCallbacks.setToolStatus(toolId, { content: message }) + return message + } + // A wildcard can match several kinds at one path (nothing in the chat + // schema could pick between them) — render each kind's section. + const sections = entries.map((entry) => renderForkEntrySection(entry, path, parent, args)) + const result = sections.join('\n\n====\n\n') + toolCallbacks.setToolStatus(toolId, { + content: `Fork vs parent diff for "${path}"`, + result + }) + return result +} + +function renderForkEntrySection( + entry: ForkDiffEntryView, + path: string, + parent: string, + args: { file?: string; offset?: number; limit?: number } +): string { + if (entry.status === 'error') { + throw new Error( + `Could not diff ${entry.kind} "${path}" against the parent: ${entry.errorMessage}` + ) + } + let draftCaveat = entry.hasLocalDraft + ? 'Note: you also have an undeployed local draft on this item — it is NOT part of this deployed-vs-deployed comparison; use diff without against to see it.\n\n' + : '' + if (entry.valueMasked && entry.status === 'modified') { + draftCaveat += + 'Note: variable values are never compared in chat — the value may also differ beyond the changes shown.\n\n' + } + const changedFileCount = entry.files ? Object.keys(entry.files).length : 0 + if (entry.status === 'unchanged' || (!entry.patch && changedFileCount === 0)) { + // A masked value can differ in content without producing a patch — + // never report that as "same content". + const message = entry.valueMasked + ? `${entry.kind} "${path}": no visible config differences vs parent "${parent}", but variable values are never shown in chat, so a value change cannot be displayed. The workspace comparison reports it as ${entry.ahead > 0 || entry.behind > 0 ? `differing (ahead ${entry.ahead}, behind ${entry.behind})` : 'in sync'}.` + : entry.kind === 'folder' + ? `folder "${path}": no comparable differences vs parent "${parent}" — the folder display name is not exposed by the API and may be what differs (comparison reports ahead ${entry.ahead}, behind ${entry.behind}).` + : `${entry.kind} "${path}" has the same content in the fork and its parent "${parent}" (only version history differs).` + return draftCaveat + message + } + const header = + entry.status === 'only_in_fork' + ? `${entry.kind} "${path}" exists only in the fork — not in parent "${parent}". Full content:\n\n` + : entry.status === 'deleted_in_fork' + ? `${entry.kind} "${path}" was deleted in the fork but still exists in parent "${parent}". Removed content:\n\n` + : `Fork changes vs parent "${parent}" for ${entry.kind} "${path}":\n\n` + if (args.file !== undefined && !entry.files) { + throw new Error( + `file only applies to multi-file apps; ${entry.kind} "${path}" diffs as a single document — call again without file.` + ) + } + const body = entry.files + ? renderEntryFiles(entry.files, entry.patch ?? '', args) + : windowPatchBody(entry.patch ?? '', args.offset ?? 0, args.limit ?? DIFF_READ_DEFAULT_LINES) + return draftCaveat + header + body +} + +const DIFF_SEARCH_DEFAULT_MAX_MATCHES = 50 +const DIFF_SEARCH_MAX_MATCHES_CEILING = 200 + +interface DiffSearchUnit { + /** Item path, or `${itemPath}/${fileName}` for a file inside an app. */ + subject: string + patch: string +} + +function collectDiffSearchUnits( + entries: Array<{ + path: string + status?: string + patch?: string + files?: Record + }>, + out: DiffSearchUnit[], + failedPaths: string[] +): void { + for (const e of entries) { + // A failed materialization has no patch — claiming "no matches" for it + // would present an incomplete search as a definitive one. + if (e.status === 'error') { + failedPaths.push(e.path) + continue + } + if (e.files) { + for (const [name, fileDiff] of Object.entries(e.files)) { + // App file keys lead with '/'; a raw join would yield `f/x//file`, + // which slash-anchored globs like `f/x/*.tsx` can never match. + out.push({ subject: `${e.path}/${name.replace(/^\/+/, '')}`, patch: fileDiff.patch }) + } + if (e.patch) out.push({ subject: e.path, patch: e.patch }) + } else if (e.patch) { + out.push({ subject: e.path, patch: e.patch }) + } + } +} + +// Literal substring search over the changed lines of every diff in the +// comparison. Materializes all patches first (search cannot skip any), then +// scans in memory — same output conventions as search_app. +async function diffSearch( + args: { + against?: 'deployed' | 'parent_workspace' + search?: string + file_glob?: string + max_matches?: number + }, + ctx: WriteDraftCtx +): Promise { + const { workspace, toolId, toolCallbacks } = ctx + const query = args.search ?? '' + if (query.length === 0) { + throw new Error('search requires a non-empty string.') + } + toolCallbacks.setToolStatus(toolId, { content: `Searching diffs for "${query}"...` }) + + const units: DiffSearchUnit[] = [] + const failedPaths: string[] = [] + let unflushedNote = '' + if (args.against === 'parent_workspace') { + const parent = forkParentOrThrow(workspace) + const index = await getForkDiffIndex(workspace, parent, { materializeAll: true }) + if (index.skippedComparison) { + const message = forkComparisonUnavailableMessage(parent) + toolCallbacks.setToolStatus(toolId, { content: message }) + return message + } + collectDiffSearchUnits(index.entries, units, failedPaths) + } else { + const { unflushedPaths } = await flushGlobalDraftSaves(workspace) + expireWorkspaceDiffList(workspace) + if (unflushedPaths.length > 0) { + unflushedNote = `\nWarning: unsaved editor changes on ${unflushedPaths.join(', ')} were not searched (auto-save off, a save failed, or a conflict is unresolved).` + } + const index = await getWorkspaceDiffIndex(workspace, { materializeAll: true }) + collectDiffSearchUnits(index.entries, units, failedPaths) + } + if (failedPaths.length > 0) { + unflushedNote += `\nWarning: ${failedPaths.length === 1 ? 'this diff' : 'these diffs'} could not be computed and ${failedPaths.length === 1 ? 'was' : 'were'} NOT searched (matches may be missing): ${failedPaths.join(', ')}. Retry, or read the item${failedPaths.length === 1 ? '' : 's'} directly for the error.` + } + const filtered = args.file_glob + ? units.filter((u) => appFileMatchesGlob(u.subject, args.file_glob as string)) + : units + + const needle = query.toLowerCase() + const maxMatches = Math.min( + args.max_matches ?? DIFF_SEARCH_DEFAULT_MAX_MATCHES, + DIFF_SEARCH_MAX_MATCHES_CEILING + ) + const matches: { subject: string; line: number; text: string }[] = [] + let totalMatchCount = 0 + let renderedMatchCount = 0 + let subjectCount = 0 + let truncated = false + for (const unit of filtered.sort((a, b) => a.subject.localeCompare(b.subject))) { + const lines = unit.patch.split('\n') + const changed = new Set(changedLineIndices(unit.patch)) + let unitHadMatch = false + for (let i = 0; i < lines.length; i++) { + if (!changed.has(i) || !lines[i].toLowerCase().includes(needle)) continue + totalMatchCount++ + unitHadMatch = true + if (renderedMatchCount >= maxMatches) { + truncated = true + continue + } + renderedMatchCount++ + const lo = Math.max(0, i - SEARCH_APP_CONTEXT_LINES) + const hi = Math.min(lines.length - 1, i + SEARCH_APP_CONTEXT_LINES) + for (let j = lo; j <= hi; j++) { + matches.push({ subject: unit.subject, line: j + 1, text: lines[j] }) + } + } + if (unitHadMatch) subjectCount++ + } + + if (totalMatchCount === 0) { + toolCallbacks.setToolStatus(toolId, { content: `No diff matches for "${query}"` }) + return ( + `No changed lines match. Try a broader or differently-spelled term${ + args.file_glob ? ', or drop the file_glob' : '' + }.` + unflushedNote + ) + } + + const header = `${totalMatchCount} changed line${totalMatchCount === 1 ? '' : 's'} match in ${subjectCount} diff${ + subjectCount === 1 ? '' : 's' + }${truncated ? ` (showing the first ${maxMatches}; narrow with file_glob or a more specific query)` : ''}` + const out: string[] = [header] + let currentSubject = '' + let budgetSpent = header.length + let budgetHit = false + const seen = new Set() + for (const m of matches) { + const dedupeKey = `${m.subject}:${m.line}` + if (seen.has(dedupeKey)) continue + seen.add(dedupeKey) + const text = + m.text.length > SEARCH_APP_MAX_LINE_CHARS + ? `${m.text.slice(0, SEARCH_APP_MAX_LINE_CHARS)}… [line truncated]` + : m.text + const subjectHeader = m.subject === currentSubject ? '' : `${m.subject}\n` + const row = `${subjectHeader} ${m.line}: ${text}` + if (budgetSpent + row.length + 1 > SEARCH_APP_TOTAL_CHAR_BUDGET) { + budgetHit = true + break + } + if (subjectHeader) currentSubject = m.subject + out.push(row) + budgetSpent += row.length + 1 + } + if (budgetHit) { + out.push( + `… output truncated at the context budget — narrow with file_glob or a more specific query.` + ) + } + + toolCallbacks.setToolStatus(toolId, { + content: `Found ${totalMatchCount} matching changed line${totalMatchCount === 1 ? '' : 's'}` + }) + return out.join('\n') + unflushedNote +} + // Flush a draft's pending editor autosave, then verify it actually landed before // the caller re-reads the persisted draft. `flush()` resolves even when the save // recorded a conflict (server has a newer version) or failed (network/5xx) — it @@ -5049,6 +6593,11 @@ async function deployDraft( } } + // Deployed state moved for EVERY branch above (some bypass + // deployDraftToWorkspace, which invalidates on its own path) — evict cached + // fork comparisons before the fallible draft cleanup below. + invalidateWorkspaceComparison(workspace) + await deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true }) // Move the chat's mask entry to the deployed path: a draft-only item's @@ -5131,6 +6680,10 @@ async function deleteWorkspaceItem( break } + // Deployed state changed — cached fork comparisons involving this workspace + // are no longer trustworthy (same rule as deploy success). Before the + // draft cleanup: a cleanup failure must not leave stale comparisons. + invalidateWorkspaceComparison(workspace) await deleteGlobalDraft(workspace, type, path, triggerKind) // Record the deletion in the chat's modified-items mask. In a fork this leaves a @@ -5248,8 +6801,52 @@ export function prepareGlobalUserMessage( content += '\n' } + const domSelectors = selectedContext.filter((c) => c.type === 'app_dom_selector') + if (domSelectors.length > 0) { + content += '## SELECTED DOM ELEMENTS\n' + content += + "The user pointed at these elements in the live raw app preview. Their HTML is not included here — inspect it live with search_dom / read_dom, passing the element's `app_path` and `selector` so the right app's preview is read.\n" + for (const el of domSelectors) { + content += `- ${el.title} — app_path: ${el.appPath}, selector: ${el.selector}\n` + } + content += '\n' + } + + const files = options.files ?? [] + if (files.length > 0) { + content += '## ATTACHED FILES\n' + content += + 'The user attached these files to this message. Their content is NOT included here — read it with `read_file` (or scan it with `search_files`), passing the file id, before answering questions about it.\n' + for (const f of files) { + // textLineCount matches read_file's numbering — a mismatch would make the + // model request line ranges past the end. + const lines = textLineCount(f.content) + // The id is the durable reference (names may repeat across messages); + // absent only on legacy pre-id transcripts, where the name resolves. + // Sanitized again here: legacy names predate attach-time sanitization. + const name = sanitizeAttachmentName(f.name) + const ref = f.id ? `${name} (file id: ${f.id})` : name + content += `- ${ref} — ${lines} lines, ${f.content.length} chars\n` + } + content += '\n' + } + content += `## INSTRUCTIONS:\n${instructions}` + const images = options.images ?? [] + if (images.length > 0) { + // Multimodal message: the text block plus one image_url part per attachment. + // The provider converters translate image_url for Anthropic/Responses; the + // OpenAI-compatible path sends it as-is. + return { + role: 'user', + content: [ + { type: 'text', text: content }, + ...images.map((img) => dataUrlToImagePart(img.dataUrl)) + ] + } + } + return { role: 'user', content diff --git a/frontend/src/lib/components/copilot/chat/global/diffSnapshot.test.ts b/frontend/src/lib/components/copilot/chat/global/diffSnapshot.test.ts new file mode 100644 index 0000000000..76aed58b7c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/diffSnapshot.test.ts @@ -0,0 +1,680 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/workspaceDrafts.svelte', () => ({ + getDraftItems: vi.fn(), + getWorkspaceDraftsVersion: vi.fn(() => 0) +})) +vi.mock('$lib/utils_draft_deploy', () => ({ + getDraftDiffValues: vi.fn() +})) +vi.mock('$lib/utils_workspace_deploy', () => ({ + getItemValue: vi.fn() +})) +vi.mock('$lib/workspaceComparison', () => ({ + fetchWorkspaceComparisonMeta: vi.fn(), + isComparisonCurrent: vi.fn(() => true) +})) +vi.mock('$lib/stores', async () => { + const { readable, writable } = await import('svelte/store') + return { + userWorkspaces: readable([{ id: 'fork-ws', parent_workspace_id: 'parent-ws' }]), + usersWorkspaceStore: writable(undefined) + } +}) +vi.mock('$lib/gen', () => ({ + VariableService: { getVariable: vi.fn() }, + ScriptService: { getScriptByPath: vi.fn() }, + ResourceService: { getResource: vi.fn(), getResourceType: vi.fn() }, + FlowService: { getFlowByPath: vi.fn() } +})) +vi.mock('./userDraftAdapter', () => ({ + itemTypeForKind: (kind: string) => + kind === 'script' + ? { type: 'script' } + : kind === 'raw_app' || kind === 'app' + ? { type: 'app' } + : kind === 'trigger_http' + ? { type: 'trigger', triggerKind: 'http' } + : undefined +})) + +import { getDraftItems, getWorkspaceDraftsVersion } from '$lib/workspaceDrafts.svelte' +import { getDraftDiffValues } from '$lib/utils_draft_deploy' +import { getItemValue } from '$lib/utils_workspace_deploy' +import { fetchWorkspaceComparisonMeta, isComparisonCurrent } from '$lib/workspaceComparison' +import { usersWorkspaceStore } from '$lib/stores' +import { FlowService, ResourceService, ScriptService, VariableService } from '$lib/gen' +import { + expireWorkspaceDiffList, + getForkDiffIndex, + getForkParentWorkspaceId, + getWorkspaceDiffIndex, + invalidateWorkspaceDiffCache, + markWorkspaceDiffEntryStale, + readForkDiffEntries, + readWorkspaceDiffEntry +} from './diffSnapshot' + +const WS = 'test-ws' + +function row(overrides: Partial> = {}) { + return { + kind: 'script', + path: 'f/a/b', + draft_only: false, + legacy_draft: false, + raw_app: false, + can_write: true, + mine: true, + created_at: '2026-07-20T00:00:00Z', + ...overrides + } +} + +function mockDiffValues(deployed: unknown, draft: unknown, noDeployed = false) { + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed, + draft, + hasDraft: true, + noDeployed + } as any) +} + +beforeEach(() => { + vi.useFakeTimers() + invalidateWorkspaceDiffCache() + vi.mocked(getDraftItems).mockReset() + vi.mocked(getDraftDiffValues).mockReset() + vi.mocked(getWorkspaceDraftsVersion).mockReset().mockReturnValue(0) +}) + +afterEach(() => { + vi.useRealTimers() +}) + +describe('getWorkspaceDiffIndex', () => { + it('materializes patches once and reuses them while rows are unchanged', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + + const first = await getWorkspaceDiffIndex(WS) + expect(first.entries).toHaveLength(1) + expect(first.entries[0].status).toBe('modified') + expect(first.entries[0].patch).toContain('-content: a') + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + + // Second access after the list-reuse window: row unchanged → no refetch. + vi.advanceTimersByTime(10_000) + const second = await getWorkspaceDiffIndex(WS) + expect(second.entries[0].status).toBe('modified') + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + }) + + it('refetches the listing after expireWorkspaceDiffList despite the reuse window', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + await getWorkspaceDiffIndex(WS) + expect(getDraftItems).toHaveBeenCalledTimes(1) + + // Within the reuse window the listing is served from cache... + await getWorkspaceDiffIndex(WS) + expect(getDraftItems).toHaveBeenCalledTimes(1) + + // ...but an expiry (post-flush) forces a refetch immediately. + expireWorkspaceDiffList(WS) + await getWorkspaceDiffIndex(WS) + expect(getDraftItems).toHaveBeenCalledTimes(2) + }) + + it('an in-flight listing fetch never serves rows that predate a mid-flight save', async () => { + let release!: () => void + const gate = new Promise((res) => (release = res)) + const oldRows = [row()] + const newRows = [row(), row({ path: 'f/a/created-mid-flight' })] + let call = 0 + vi.mocked(getDraftItems).mockImplementation(async () => { + call++ + if (call === 1) { + await gate + return oldRows as any + } + return newRows as any + }) + mockDiffValues({ content: 'a' }, { content: 'b' }) + + const first = getWorkspaceDiffIndex(WS) + // A save lands while the first listing fetch is in flight... + markWorkspaceDiffEntryStale(WS, 'script', 'f/a/created-mid-flight') + release() + const index = await first + // ...so the result must reflect the post-save listing, not the stale rows. + expect(index.entries.map((e) => e.storagePath)).toContain('f/a/created-mid-flight') + expect(getDraftItems).toHaveBeenCalledTimes(2) + }) + + it('refetches a stale-marked entry immediately, despite every reuse window', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + await readWorkspaceDiffEntry(WS, 'script', 'f/a/b') + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + + // Within both the list throttle and the read-reuse window the patch is + // served from cache... + await readWorkspaceDiffEntry(WS, 'script', 'f/a/b') + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + + // ...but a landed save (the syncer hook) forces content + listing fresh. + markWorkspaceDiffEntryStale(WS, 'script', 'f/a/b') + mockDiffValues({ content: 'a' }, { content: 'c' }) + const entry = await readWorkspaceDiffEntry(WS, 'script', 'f/a/b') + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + expect(entry?.patch).toContain('+content: c') + }) + + it('refetches an entry when its draft row created_at changes', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + await getWorkspaceDiffIndex(WS) + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + + vi.advanceTimersByTime(10_000) + vi.mocked(getDraftItems).mockResolvedValue([row({ created_at: '2026-07-20T01:00:00Z' })] as any) + mockDiffValues({ content: 'a' }, { content: 'a' }) + const index = await getWorkspaceDiffIndex(WS) + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + expect(index.entries[0].status).toBe('unchanged') + }) + + it('drops cached patches when the workspace drafts version bumps', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + await getWorkspaceDiffIndex(WS) + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + + vi.mocked(getWorkspaceDraftsVersion).mockReturnValue(1) + await getWorkspaceDiffIndex(WS) + expect(getDraftDiffValues).toHaveBeenCalledTimes(2) + }) + + it('excludes other users drafts from entries but counts them', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row(), + row({ path: 'f/other/x', mine: false }) + ] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + const index = await getWorkspaceDiffIndex(WS) + expect(index.entries).toHaveLength(1) + expect(index.otherUsersDraftCount).toBe(1) + }) + + it('maps statuses: never-deployed → new, unaddressable kind → not_diffable', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ path: 'f/a/new', draft_only: true }), + row({ kind: 'data_pipeline', path: 'f/a/pipe' }) + ] as any) + mockDiffValues(undefined, { content: 'b' }, true) + const index = await getWorkspaceDiffIndex(WS) + const byPath = Object.fromEntries(index.entries.map((e) => [e.storagePath, e])) + expect(byPath['f/a/new'].status).toBe('new') + expect(byPath['f/a/pipe'].status).toBe('not_diffable') + // Unaddressable kinds are never fetched. + expect(getDraftDiffValues).toHaveBeenCalledTimes(1) + }) + + it('masks draft-mode variable values but still marks a value change', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'variable', path: 'f/a/token' }) + ] as any) + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: { value: 'old-plaintext', is_secret: false, description: 'd' }, + draft: { value: 'new-plaintext', is_secret: false, description: 'd' }, + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'variable', 'f/a/token') + expect(entry?.status).toBe('modified') + expect(entry?.valueMasked).toBe(true) + // No plaintext on either side — the patch only marks that it changed. + expect(entry?.patch).not.toContain('plaintext') + expect(entry?.patch).toContain('(changed)') + }) + + it('flags a secret-only draft as uncomparable instead of claiming unchanged', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'variable', path: 'f/a/secret' }) + ] as any) + // The shared canonicalizer already masked both sides to the same + // sentinel — equality between them proves nothing. + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: { value: '', is_secret: true, description: 'd' }, + draft: { value: '', is_secret: true, description: 'd' }, + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'variable', 'f/a/secret') + expect(entry?.status).toBe('unchanged') + expect(entry?.valueUncomparable).toBe(true) + }) + + it('keeps the uncomparable flag on a secret whose metadata also changed', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'variable', path: 'f/a/secret' }) + ] as any) + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: { value: '', is_secret: true, description: 'old desc' }, + draft: { value: '', is_secret: true, description: 'new desc' }, + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'variable', 'f/a/secret') + // A non-empty metadata patch must not swallow the secret caveat: the + // value itself may ALSO differ beyond what the patch shows. + expect(entry?.status).toBe('modified') + expect(entry?.valueUncomparable).toBe(true) + expect(entry?.patch).toContain('desc') + expect(entry?.patch).not.toContain(' { + vi.mocked(getDraftItems).mockResolvedValue([row({ kind: 'app', path: 'f/a/classic' })] as any) + mockDiffValues({ summary: 'v1' }, { summary: 'v2' }) + const index = await getWorkspaceDiffIndex(WS) + expect(index.entries[0].type).toBe('app') + expect(index.entries[0].status).toBe('modified') + }) + + it('leaves entries beyond the eager cap pending', async () => { + const rows = Array.from({ length: 55 }, (_, i) => row({ path: `f/a/s${i}` })) + vi.mocked(getDraftItems).mockResolvedValue(rows as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + const index = await getWorkspaceDiffIndex(WS) + expect(index.entries).toHaveLength(55) + expect(index.entries.filter((e) => e.status === 'pending')).toHaveLength(5) + expect(getDraftDiffValues).toHaveBeenCalledTimes(50) + }) +}) + +describe('raw-app file splitting (draft mode)', () => { + it('produces per-file patches plus a config-only patch', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'raw_app', path: 'f/dash/main' }) + ] as any) + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: { + summary: 'Dash', + files: { 'src/App.tsx': 'a\nb\nc\n', 'old.js': 'legacy\n' }, + runnables: {} + }, + draft: { + summary: 'Dash v2', + files: { 'src/App.tsx': 'a\nB\nc\n', 'new.ts': 'fresh\n' }, + runnables: {} + }, + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'raw_app', 'f/dash/main') + expect(entry?.status).toBe('modified') + expect(entry?.files?.['src/App.tsx'].status).toBe('modified') + expect(entry?.files?.['src/App.tsx'].patch).toContain('-b') + expect(entry?.files?.['new.ts'].status).toBe('added') + expect(entry?.files?.['old.js'].status).toBe('deleted') + // Config patch carries the summary change but no file contents. + expect(entry?.patch).toContain('summary') + expect(entry?.patch).not.toContain('src/App.tsx') + }) + + it('reports empty-file additions and deletions as changes despite the empty patch', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'raw_app', path: 'f/dash/main' }) + ] as any) + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: { summary: 'Dash', files: { 'gone.css': '', 'kept.ts': 'same\n' }, runnables: {} }, + draft: { summary: 'Dash', files: { 'added.ts': '', 'kept.ts': 'same\n' }, runnables: {} }, + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'raw_app', 'f/dash/main') + expect(entry?.status).toBe('modified') + expect(entry?.files?.['added.ts']).toEqual({ status: 'added', patch: '', lineCount: 0 }) + expect(entry?.files?.['gone.css']).toEqual({ status: 'deleted', patch: '', lineCount: 0 }) + expect(entry?.files?.['kept.ts']).toBeUndefined() + }) + + it('reports unchanged when neither files nor config differ', async () => { + const value = { summary: 'Dash', files: { 'a.ts': 'same\n' }, runnables: {} } + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'raw_app', path: 'f/dash/main' }) + ] as any) + vi.mocked(getDraftDiffValues).mockResolvedValue({ + deployed: value, + draft: structuredClone(value), + hasDraft: true, + noDeployed: false + } as any) + const entry = await readWorkspaceDiffEntry(WS, 'raw_app', 'f/dash/main') + expect(entry?.status).toBe('unchanged') + expect(entry?.files).toEqual({}) + }) +}) + +describe('readWorkspaceDiffEntry', () => { + it('resolves a draft-only item by its friendly draft_path', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ path: 'u/admin/draft_123', draft_path: 'f/nice/name', draft_only: true }) + ] as any) + mockDiffValues(undefined, { summary: 'x' }, true) + const entry = await readWorkspaceDiffEntry(WS, 'script', 'f/nice/name') + expect(entry?.storagePath).toBe('u/admin/draft_123') + expect(entry?.status).toBe('new') + }) + + // (resolveWorkspaceDiffTarget) + it('resolves a renamed draft to its owning row across kinds', async () => { + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'app', path: 'f/old/name', draft_path: 'f/new/name' }) + ] as any) + const { resolveWorkspaceDiffTarget } = await import('./diffSnapshot') + const target = await resolveWorkspaceDiffTarget(WS, ['raw_app', 'app'], 'f/new/name') + expect(target).toEqual({ kind: 'app', storagePath: 'f/old/name' }) + }) + + it('returns undefined when the user has no draft at the path', async () => { + vi.mocked(getDraftItems).mockResolvedValue([] as any) + const entry = await readWorkspaceDiffEntry(WS, 'script', 'f/a/b') + expect(entry).toBeUndefined() + expect(getDraftDiffValues).not.toHaveBeenCalled() + }) + + it('records a fetch failure as an error entry instead of throwing', async () => { + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + vi.mocked(getDraftDiffValues).mockRejectedValue({ status: 404 }) + const entry = await readWorkspaceDiffEntry(WS, 'script', 'f/a/b') + expect(entry?.status).toBe('error') + expect(entry?.errorMessage).toContain('not found') + }) +}) + +async function readForkDiffEntryOne( + workspace: string, + parent: string, + kinds: string[], + path: string +) { + const entries = await readForkDiffEntries(workspace, parent, kinds, path) + return entries[0] +} + +const FORK = 'fork-ws' +const PARENT = 'parent-ws' + +function comparisonDiff(overrides: Partial> = {}) { + return { + kind: 'schedule', + path: 'f/a/b', + ahead: 1, + behind: 0, + has_changes: true, + exists_in_source: true, + exists_in_fork: true, + ...overrides + } +} + +let comparisonGen = 0 +function mockComparison(diffs: unknown[], meta: { fetchedAt?: number } = {}) { + const fetchedAt = meta.fetchedAt + vi.mocked(fetchWorkspaceComparisonMeta).mockImplementation(async () => ({ + comparison: { + skipped_comparison: false, + diffs, + summary: { total_diffs: diffs.length }, + hidden_ahead: { total: 0, by_kind: {}, items: [] }, + hidden_behind: { total: 0, by_kind: {}, items: [] } + } as any, + fetchedAt: fetchedAt ?? Date.now(), + generation: ++comparisonGen + })) +} + +describe('fork mode', () => { + beforeEach(() => { + vi.mocked(getDraftItems).mockResolvedValue([] as any) + vi.mocked(getItemValue).mockReset() + vi.mocked(fetchWorkspaceComparisonMeta).mockReset() + vi.mocked(isComparisonCurrent).mockReset().mockReturnValue(true) + vi.mocked(VariableService.getVariable).mockReset() + }) + + it('resolves the parent workspace from the user workspaces store', () => { + expect(getForkParentWorkspaceId(FORK)).toBe(PARENT) + expect(getForkParentWorkspaceId('plain-ws')).toBeUndefined() + }) + + it('diffs both deployed sides and maps one-sided items', async () => { + mockComparison([ + comparisonDiff(), + comparisonDiff({ path: 'f/a/new', exists_in_source: false }), + comparisonDiff({ path: 'f/a/gone', exists_in_fork: false, ahead: 0, behind: 1 }) + ]) + vi.mocked(getItemValue).mockImplementation(async (_k, path, ws) => ({ + content: `${path}@${ws}` + })) + const index = await getForkDiffIndex(FORK, PARENT) + const byPath = Object.fromEntries(index.entries.map((e) => [e.path, e])) + expect(byPath['f/a/b'].status).toBe('modified') + expect(byPath['f/a/new'].status).toBe('only_in_fork') + expect(byPath['f/a/new'].patch).not.toContain('parent-ws') + expect(byPath['f/a/gone'].status).toBe('deleted_in_fork') + // One-sided entries fetch only the existing side: 2 + 1 + 1 calls. + expect(getItemValue).toHaveBeenCalledTimes(4) + }) + + it('ages a reused comparison from its fetch time, not adoption time', async () => { + // Adopt a tally another surface fetched 25s ago (tolerant join). + mockComparison([comparisonDiff()], { fetchedAt: Date.now() - 25_000 }) + vi.mocked(getItemValue).mockResolvedValue({ content: 'x' }) + await getForkDiffIndex(FORK, PARENT) + expect(fetchWorkspaceComparisonMeta).toHaveBeenCalledTimes(1) + // 10s later the underlying tally is 35s old — past the reuse window. + // The snapshot must not have granted it a fresh window of its own. + vi.advanceTimersByTime(10_000) + await getForkDiffIndex(FORK, PARENT) + expect(fetchWorkspaceComparisonMeta).toHaveBeenCalledTimes(2) + }) + + it('drops the snapshot when the comparison store was invalidated without a drafts bump', async () => { + mockComparison([comparisonDiff()]) + vi.mocked(getItemValue).mockResolvedValue({ content: 'x' }) + await getForkDiffIndex(FORK, PARENT) + expect(fetchWorkspaceComparisonMeta).toHaveBeenCalledTimes(1) + // A deploy invalidated the comparison store but its draft cleanup + // failed, so no drafts-version bump ever reaches this cache. + vi.mocked(isComparisonCurrent).mockReturnValue(false) + await getForkDiffIndex(FORK, PARENT) + expect(fetchWorkspaceComparisonMeta).toHaveBeenCalledTimes(2) + }) + + it('reports a swallowed side fetch as an error, never as a fabricated diff', async () => { + mockComparison([comparisonDiff()]) + // The shared reader returns {} for ANY failed fetch — with both sides + // "empty" a real diff would read as parity, one side as an addition. + vi.mocked(getItemValue).mockResolvedValue({}) + const index = await getForkDiffIndex(FORK, PARENT) + expect(index.entries[0].status).toBe('error') + expect(index.entries[0].errorMessage).toContain('failed to read') + }) + + it('never joins a fork reconciliation started under a previous account', async () => { + usersWorkspaceStore.set({ email: 'fork-a@x.dev' } as any) + mockComparison([comparisonDiff({ path: 'f/a/fresh' })]) + let resolveStale!: (v: unknown) => void + vi.mocked(fetchWorkspaceComparisonMeta).mockReturnValueOnce( + new Promise((res) => (resolveStale = res)) as any + ) + vi.mocked(getItemValue).mockResolvedValue({ content: 'x' }) + const staleRead = getForkDiffIndex(FORK, PARENT) + usersWorkspaceStore.set({ email: 'fork-b@x.dev' } as any) + const after = await getForkDiffIndex(FORK, PARENT) + expect(after.entries.map((e) => e.path)).toEqual(['f/a/fresh']) + resolveStale({ + comparison: { + skipped_comparison: false, + diffs: [comparisonDiff({ path: 'f/a/stale' })], + summary: { total_diffs: 1 }, + hidden_ahead: { total: 0 }, + hidden_behind: { total: 0 } + }, + fetchedAt: Date.now(), + generation: ++comparisonGen + }) + // The old account's read resolves too — it must also get the fresh + // snapshot, never its own late (pre-switch) tally. + expect((await staleRead).entries.map((e) => e.path)).toEqual(['f/a/fresh']) + const reread = await getForkDiffIndex(FORK, PARENT) + expect(reread.entries.map((e) => e.path)).toEqual(['f/a/fresh']) + }) + + it('reuses patches while the item ahead/behind marker is unchanged, refetches when it moves', async () => { + mockComparison([comparisonDiff()]) + vi.mocked(getItemValue).mockResolvedValue({ content: 'x' }) + await getForkDiffIndex(FORK, PARENT) + expect(getItemValue).toHaveBeenCalledTimes(2) + + // Past the comparison reuse window, same marker → comparison refetched + // but the patch is kept. + vi.advanceTimersByTime(31_000) + await getForkDiffIndex(FORK, PARENT) + expect(getItemValue).toHaveBeenCalledTimes(2) + + // Marker moved → content refetched. + vi.advanceTimersByTime(31_000) + mockComparison([comparisonDiff({ ahead: 2 })]) + await getForkDiffIndex(FORK, PARENT) + expect(getItemValue).toHaveBeenCalledTimes(4) + }) + + it('never decrypts variables and masks every value, secret or not', async () => { + mockComparison([comparisonDiff({ kind: 'variable', path: 'f/a/plain' })]) + vi.mocked(VariableService.getVariable).mockImplementation(async ({ workspace }: any) => ({ + path: 'f/a/plain', + is_secret: false, + value: `plaintext-${workspace}`, + description: workspace === FORK ? 'fork side' : 'parent side' + })) as any + const entry = await readForkDiffEntryOne(FORK, PARENT, ['variable'], 'f/a/plain') + expect( + vi.mocked(VariableService.getVariable).mock.calls.every(([a]: any) => !a.decryptSecret) + ).toBe(true) + // NON-secret values are masked too — the chat never sees variable values. + expect(entry?.patch).not.toContain('plaintext-') + // Description change still shows; the value itself never differs here. + expect(entry?.patch).toContain('fork side') + expect(entry?.valueMasked).toBe(true) + }) + + it('sees flow schema-only changes the shared projection drops', async () => { + mockComparison([comparisonDiff({ kind: 'flow', path: 'f/a/fl' })]) + vi.mocked(FlowService.getFlowByPath).mockImplementation(async ({ workspace }: any) => ({ + summary: 'same', + description: 'same', + schema: { properties: workspace === FORK ? { a: {} } : { a: {}, b: {} } }, + value: { modules: [{ id: 'x', value: { type: 'script', hash: `h-${workspace}` } }] } + })) as any + const entry = await readForkDiffEntryOne(FORK, PARENT, ['flow'], 'f/a/fl') + expect(entry?.status).toBe('modified') + expect(entry?.patch).toContain('schema') + // Inline-script hashes are per-workspace noise, never a diff line. + expect(entry?.patch).not.toContain('h-fork-ws') + }) + + it('sees script description-only changes the shared projection drops', async () => { + mockComparison([comparisonDiff({ kind: 'script', path: 'f/a/s' })]) + vi.mocked(ScriptService.getScriptByPath).mockImplementation(async ({ workspace }: any) => ({ + content: 'same code', + summary: 'same', + description: workspace === FORK ? 'fork docs' : 'parent docs', + language: 'bun' + })) as any + const entry = await readForkDiffEntryOne(FORK, PARENT, ['script'], 'f/a/s') + expect(entry?.status).toBe('modified') + expect(entry?.patch).toContain('-description: parent docs') + }) + + it('splits a raw app into per-file patches with cross-workspace noise stripped', async () => { + mockComparison([comparisonDiff({ kind: 'raw_app', path: 'f/dash/main' })]) + vi.mocked(getItemValue).mockImplementation(async (_k, _path, ws) => ({ + raw_app: true, + summary: 'Dash', + versions: ws === FORK ? [7] : [3], + workspace_id: ws, + value: { + files: + ws === FORK + ? { 'src/App.tsx': 'new content\n', 'src/added.ts': 'brand new\n' } + : { 'src/App.tsx': 'old content\n', 'src/removed.ts': 'gone\n' }, + runnables: {} + } + })) + const entry = await readForkDiffEntryOne(FORK, PARENT, ['app', 'raw_app'], 'f/dash/main') + expect(entry?.status).toBe('modified') + expect(entry?.files?.['src/App.tsx'].status).toBe('modified') + expect(entry?.files?.['src/App.tsx'].patch).toContain('-old content') + expect(entry?.files?.['src/added.ts'].status).toBe('added') + expect(entry?.files?.['src/removed.ts'].status).toBe('deleted') + // Version counters and workspace ids never appear as config changes. + expect(entry?.patch).not.toContain('versions') + expect(entry?.patch).not.toContain('workspace_id') + expect(entry?.patch).not.toContain('parent_version') + }) + + it('reads fork-only kinds by path alone; a wildcard returns every matching kind', async () => { + mockComparison([ + comparisonDiff({ kind: 'folder', path: 'f/team' }), + comparisonDiff({ kind: 'resource_type', path: 'shared_name' }), + comparisonDiff({ kind: 'folder', path: 'shared_name' }) + ]) + vi.mocked(getItemValue).mockImplementation(async (_k, path, ws) => ({ + name: path, + summary: `${ws}` + })) + vi.mocked(ResourceService.getResourceType).mockImplementation(async ({ workspace }: any) => ({ + schema: {}, + description: `${workspace}` + })) as any + const folder = await readForkDiffEntryOne(FORK, PARENT, [], 'f/team') + expect(folder?.kind).toBe('folder') + expect(folder?.status).toBe('modified') + + const both = await readForkDiffEntries(FORK, PARENT, [], 'shared_name') + expect(both.map((e) => e.kind).sort()).toEqual(['folder', 'resource_type']) + expect(both.every((e) => e.status === 'modified')).toBe(true) + }) + + it('flags entries that also carry a local draft', async () => { + mockComparison([comparisonDiff(), comparisonDiff({ path: 'f/a/other' })]) + vi.mocked(getDraftItems).mockResolvedValue([ + row({ kind: 'trigger_schedule', path: 'f/a/b' }) + ] as any) + vi.mocked(getItemValue).mockResolvedValue({ content: 'x' }) + const index = await getForkDiffIndex(FORK, PARENT) + const byPath = Object.fromEntries(index.entries.map((e) => [e.path, e])) + expect(byPath['f/a/b'].hasLocalDraft).toBe(true) + expect(byPath['f/a/other'].hasLocalDraft).toBe(false) + }) +}) + +describe('account switches', () => { + it("never serves one account's cached drafts to another", async () => { + usersWorkspaceStore.set({ email: 'first@x.dev' } as any) + vi.mocked(getDraftItems).mockResolvedValue([row()] as any) + mockDiffValues({ content: 'a' }, { content: 'b' }) + await getWorkspaceDiffIndex(WS) + await getWorkspaceDiffIndex(WS) + // Same account: the throttled listing is reused. + expect(getDraftItems).toHaveBeenCalledTimes(1) + usersWorkspaceStore.set({ email: 'second@x.dev' } as any) + await getWorkspaceDiffIndex(WS) + expect(getDraftItems).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/diffSnapshot.ts b/frontend/src/lib/components/copilot/chat/global/diffSnapshot.ts new file mode 100644 index 0000000000..f8a685af60 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/diffSnapshot.ts @@ -0,0 +1,1141 @@ +/** + * Materialized draft-vs-deployed diff cache for the chat `diff` tool. + * + * The backend has no diff endpoint — it serves item sides, the frontend + * computes patches. This module fetches each changed item's sides once, + * computes the stable-YAML patch once, and answers every subsequent query + * (workspace index, item read, later: search) from memory. + * + * Freshness model, cheapest signal first: + * - `drafts/list` is refetched per access (throttled) — one indexed query that + * yields the authoritative row set. The server bumps a draft row's + * `created_at` on every update, so an unchanged (path, created_at) pair + * proves the draft side of a cached patch is current. + * - `getWorkspaceDraftsVersion` bumps on every in-app deploy/discard/draft + * write; a bump drops cached patches because the deployed side may have + * changed without touching any draft row. + * - Deploys from OTHER clients are invisible to both signals, so index + * accesses also refresh patches older than `INDEX_ENTRY_STALE_MS`. + */ +import { get } from 'svelte/store' +import { + getDraftItems, + getWorkspaceDraftsVersion, + type DraftItem +} from '$lib/workspaceDrafts.svelte' +import { + FlowService, + ResourceService, + ScriptService, + VariableService, + type UserDraftItemKind, + type WorkspaceItemDiff +} from '$lib/gen' +import { getDraftDiffValues } from '$lib/utils_draft_deploy' +import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' +import { getItemValue } from '$lib/utils_workspace_deploy' +import type { Kind as DeployKind } from '$lib/utils_deployable' +import { userWorkspaces, usersWorkspaceStore } from '$lib/stores' +import { fetchWorkspaceComparisonMeta, isComparisonCurrent } from '$lib/workspaceComparison' +import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' +import { textFilePatch, yamlValuePatch } from './draftDiff' +import { itemTypeForKind } from './userDraftAdapter' +import { TRIGGER_KINDS, type TriggerKind, type WorkspaceItemType } from './workspaceItems' + +const LIST_REUSE_MS = 5_000 +const INDEX_ENTRY_STALE_MS = 60_000 +const READ_ENTRY_REUSE_MS = 15_000 +/** Max patches computed per index access, so a workspace with hundreds of + * drafts still answers its first index quickly; the rest report `pending` + * and materialize on item read. */ +const EAGER_MATERIALIZE_CAP = 50 +const FETCH_CONCURRENCY = 6 + +export type WorkspaceDiffStatus = + | 'new' + | 'modified' + | 'unchanged' + | 'pending' + | 'error' + | 'not_diffable' + +/** One changed file inside a multi-file (raw) app. */ +export interface DiffFileView { + status: 'added' | 'deleted' | 'modified' + patch: string + lineCount: number +} + +export interface WorkspaceDiffEntryView { + kind: UserDraftItemKind + /** Chat-facing addressing; undefined when the chat cannot address the kind. */ + type?: WorkspaceItemType + triggerKind?: TriggerKind + /** Friendly path the model should use (draft_path when present). */ + path: string + storagePath: string + summary?: string + status: WorkspaceDiffStatus + /** Unified patch; present once materialized ('' when unchanged). For a + * multi-file app this is the config-only patch — file contents live in + * `files`. */ + patch?: string + patchLineCount?: number + /** Per-file patches for multi-file apps (changed files only). */ + files?: Record + /** Variable content was placeholder-masked (values never reach the chat); + * the patch marks a value change without revealing it. */ + valueMasked?: boolean + /** Secret variable: neither side's real value is readable, so "no visible + * changes" cannot prove the value is unchanged. */ + valueUncomparable?: boolean + /** True when the item has never been deployed — the whole draft is new. */ + noDeployed?: boolean + errorMessage?: string +} + +export interface WorkspaceDiffIndexView { + entries: WorkspaceDiffEntryView[] + otherUsersDraftCount: number +} + +interface Materialized { + status: 'new' | 'modified' | 'unchanged' | 'error' + patch: string + lineCount: number + files?: Record + valueMasked?: boolean + valueUncomparable?: boolean + noDeployed: boolean + errorMessage?: string + fetchedAt: number +} + +/** `files` maps of string contents mark a multi-file app value; anything else + * (classic apps, other kinds) diffs as one document. */ +function extractAppFiles(value: unknown): Record | undefined { + const files = (value as { files?: unknown } | null | undefined)?.files + if (files == null || typeof files !== 'object' || Array.isArray(files)) return undefined + const entries = Object.entries(files as Record) + if (entries.length === 0 || !entries.every(([, v]) => typeof v === 'string')) return undefined + return files as Record +} + +interface AppSplit { + files: Record + configPatch: string + totalLines: number + hasChanges: boolean +} + +/** Split a multi-file app diff into per-file text patches plus a config-only + * YAML patch, so code diffs read file-by-file and file addressing works. + * Returns undefined when neither side carries a file map. */ +function computeAppSplit( + before: unknown, + after: unknown, + beforeLabel: string, + afterLabel: string +): AppSplit | undefined { + const beforeFiles = extractAppFiles(before) + const afterFiles = extractAppFiles(after) + if (!beforeFiles && !afterFiles) return undefined + const files: Record = {} + let totalLines = 0 + const names = [ + ...new Set([...Object.keys(beforeFiles ?? {}), ...Object.keys(afterFiles ?? {})]) + ].sort() + for (const name of names) { + const beforeContent = beforeFiles?.[name] + const afterContent = afterFiles?.[name] + const patch = textFilePatch(beforeContent, afterContent, beforeLabel, afterLabel) + // An EMPTY file appearing or disappearing yields no text patch, but file + // presence is a change in its own right (imports/bundling see it). + const presenceChanged = (beforeContent === undefined) !== (afterContent === undefined) + if (!patch && !presenceChanged) continue + const lineCount = patch === '' ? 0 : patch.split('\n').length + totalLines += lineCount + files[name] = { + status: + beforeContent === undefined ? 'added' : afterContent === undefined ? 'deleted' : 'modified', + patch, + lineCount + } + } + const withoutFiles = (value: unknown) => { + if (value == null || typeof value !== 'object') return value + const { files: _files, ...rest } = value as Record + return rest + } + const configPatch = yamlValuePatch( + withoutFiles(before), + withoutFiles(after), + beforeLabel, + afterLabel + ) + totalLines += configPatch === '' ? 0 : configPatch.split('\n').length + return { + files, + configPatch, + totalLines, + hasChanges: Object.keys(files).length > 0 || configPatch !== '' + } +} + +export interface DiffParts { + /** Whole-value patch, or the config-only patch for a multi-file app. */ + patch: string + files?: Record + lineCount: number + hasChanges: boolean +} + +/** Patch parts for a before/after value pair: per-file text patches for + * multi-file apps, one whole-value YAML patch otherwise. */ +export function computeDiffParts( + before: unknown, + after: unknown, + beforeLabel: string, + afterLabel: string +): DiffParts { + const split = computeAppSplit(before, after, beforeLabel, afterLabel) + if (split) { + return { + patch: split.configPatch, + files: split.files, + lineCount: split.totalLines, + hasChanges: split.hasChanges + } + } + const patch = yamlValuePatch(before, after, beforeLabel, afterLabel) + return { + patch, + lineCount: patch === '' ? 0 : patch.split('\n').length, + hasChanges: patch !== '' + } +} + +interface CacheEntry { + row: DraftItem + type?: WorkspaceItemType + triggerKind?: TriggerKind + displayPath: string + materialized?: Materialized + materializing?: Promise + /** Bumped by stale-marking; a materialization started before the bump + * must not store its (pre-save) result. */ + staleGeneration?: number +} + +interface WorkspaceCache { + version: number + epoch: number + listFetchedAt: number + entries: Map + otherUsersDraftCount: number +} + +const caches = new Map() +const reconciling = new Map>() +// Bumped on every invalidation. Producers capture it at start and refuse to +// store results whose inputs predate a mutation — ONE fence for every async +// producer instead of bespoke per-surface races. +const mutationEpochs = new Map() + +function mutationEpoch(workspace: string): number { + return mutationEpochs.get(workspace) ?? 0 +} + +function bumpMutationEpoch(workspace: string): void { + mutationEpochs.set(workspace, mutationEpoch(workspace) + 1) +} + +// Caches hold per-user drafts and permission-filtered fork patches but are +// keyed only by workspace — an SPA logout/login must never serve one +// account's content to another. The epoch bumps fence in-flight producers +// started under the previous identity. +let cacheOwner: string | undefined = undefined + +function ensureCacheOwner(): void { + const owner = get(usersWorkspaceStore)?.email + if (owner === cacheOwner) return + cacheOwner = owner + for (const ws of new Set([ + ...caches.keys(), + ...forkCaches.keys(), + ...reconciling.keys(), + ...forkReconciling.keys() + ])) { + bumpMutationEpoch(ws) + } + caches.clear() + forkCaches.clear() + reconciling.clear() + forkReconciling.clear() +} + +function entryKey(kind: UserDraftItemKind, storagePath: string): string { + return `${kind}:${storagePath}` +} + +/** Expire the throttled drafts listing for a workspace so the next access + * refetches it. Called after a flush persists edits: a flush bumps neither the + * drafts version nor a row's cached `created_at`, so within `LIST_REUSE_MS` + * the diff would otherwise be computed from the pre-flush listing. Cached + * patches survive — reconciliation drops exactly the rows whose `created_at` + * moved. */ +export function expireWorkspaceDiffList(workspace: string): void { + const cache = caches.get(workspace) + if (cache) cache.listFetchedAt = 0 + bumpMutationEpoch(workspace) +} + +/** Mark one item's cached patch stale and expire the listing throttle, so the + * next access refetches both — regardless of the reuse windows. Driven by the + * syncer's save hook: the moment a draft write lands (an editor autosave, a + * chat write, a delete), the pre-write patch must never be served again. */ +export function markWorkspaceDiffEntryStale( + workspace: string, + itemKind: UserDraftItemKind, + path: string +): void { + bumpMutationEpoch(workspace) + const cache = caches.get(workspace) + if (cache) { + cache.listFetchedAt = 0 + const entry = cache.entries.get(entryKey(itemKind, path)) + if (entry) { + entry.materialized = undefined + entry.staleGeneration = (entry.staleGeneration ?? 0) + 1 + } + } + // Fork entries embed a hasLocalDraft flag joined from the draft rows — + // refresh the join on the next fork access (patches survive via markers). + const fork = forkCaches.get(workspace) + if (fork) fork.fetchedAt = 0 +} + +// One app-lifetime subscription: every persisted draft write invalidates its +// item eagerly instead of waiting out the listing throttle / read-reuse +// windows. Fork caches are untouched — they compare deployed sides only. +UserDraftDbSyncer.onAnySaved(({ workspace, itemKind, path }) => { + markWorkspaceDiffEntryStale(workspace, itemKind, path) +}) + +export function invalidateWorkspaceDiffCache(workspace?: string): void { + if (workspace === undefined) { + caches.clear() + reconciling.clear() + forkCaches.clear() + forkReconciling.clear() + } else { + caches.delete(workspace) + reconciling.delete(workspace) + forkCaches.delete(workspace) + forkReconciling.delete(workspace) + } +} + +async function mapPool( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + const queue = [...items] + const workers = Array.from({ length: Math.min(limit, queue.length) }, async () => { + for (let item = queue.shift(); item !== undefined; item = queue.shift()) { + await fn(item) + } + }) + await Promise.all(workers) +} + +/** Refetch the workspace draft rows and rebuild the entry map, carrying over + * each cached patch whose draft row is provably unchanged. */ +async function reconcile(workspace: string): Promise { + ensureCacheOwner() + // Retry loop: a joiner re-validates after awaiting, and a producer refuses + // to store results whose inputs predate a mutation (epoch moved mid-fetch). + // Bounded: each retry needs another concurrent mutation, and the final + // attempt's result is served regardless so a save storm cannot livelock us. + for (let attempt = 0; ; attempt++) { + const prev = caches.get(workspace) + const version = getWorkspaceDraftsVersion(workspace) + const epoch = mutationEpoch(workspace) + if ( + prev && + prev.version === version && + prev.epoch === epoch && + Date.now() - prev.listFetchedAt < LIST_REUSE_MS + ) { + return prev + } + const inflight = reconciling.get(workspace) + if (inflight) { + const joined = await inflight + if (joined.epoch === mutationEpoch(workspace) || attempt >= 4) return joined + continue + } + const run = (async () => { + const rows = await getDraftItems(workspace, true) + const mine = rows.filter((r) => r.mine) + const entries = new Map() + for (const row of mine) { + const key = entryKey(row.kind, row.path) + const old = prev?.entries.get(key) + const reusable = + old && prev!.version === version && old.row.created_at === row.created_at + ? old.materialized + : undefined + const addressing = itemTypeForKind(row.kind) + entries.set(key, { + row, + type: addressing?.type, + triggerKind: addressing?.triggerKind, + displayPath: row.draft_path || row.path, + materialized: reusable + }) + } + return { + version, + epoch, + listFetchedAt: Date.now(), + entries, + otherUsersDraftCount: rows.length - mine.length + } satisfies WorkspaceCache + })() + reconciling.set(workspace, run) + let cache: WorkspaceCache + try { + cache = await run + } finally { + if (reconciling.get(workspace) === run) reconciling.delete(workspace) + } + if (mutationEpoch(workspace) === epoch || attempt >= 4) { + caches.set(workspace, cache) + return cache + } + // Inputs predate a mutation that landed mid-fetch: refetch. + } +} + +const VARIABLE_VALUE_PLACEHOLDER = '' +const VARIABLE_VALUE_CHANGED_PLACEHOLDER = '' + +/** Chat-side redaction: variable VALUES never reach a tool result — the same + * invariant read_workspace_item enforces, and not limited to secrets. The + * placeholder pair still shows WHETHER the value changed, never its content. */ +export function maskVariableDiffSides( + before: unknown, + after: unknown +): { before: unknown; after: unknown; valueUncomparable: boolean } { + const beforeObj = + before !== null && typeof before === 'object' ? (before as Record) : undefined + const afterObj = + after !== null && typeof after === 'object' ? (after as Record) : undefined + // A secret's sides are already masked upstream (draft rows store '' and the + // deployed value is never decrypted), so equality between them proves + // nothing — the value may have changed invisibly. Only non-secret sides + // carry real content worth comparing. + const secret = beforeObj?.is_secret === true || afterObj?.is_secret === true + const valueUncomparable = secret && beforeObj !== undefined && afterObj !== undefined + const valueChanged = + !secret && + beforeObj !== undefined && + afterObj !== undefined && + JSON.stringify(beforeObj.value) !== JSON.stringify(afterObj.value) + return { + before: beforeObj ? { ...beforeObj, value: VARIABLE_VALUE_PLACEHOLDER } : before, + after: afterObj + ? { + ...afterObj, + value: valueChanged ? VARIABLE_VALUE_CHANGED_PLACEHOLDER : VARIABLE_VALUE_PLACEHOLDER + } + : after, + valueUncomparable + } +} + +/** Fetch one entry's sides and compute its patch, deduping concurrent calls. + * Reuses the cached patch when younger than `maxAgeMs`. */ +async function materialize(workspace: string, entry: CacheEntry, maxAgeMs: number): Promise { + if (entry.materialized && Date.now() - entry.materialized.fetchedAt < maxAgeMs) return + if (entry.materializing) { + // If a save landed mid-flight, the awaited run discarded its result — + // fall through and fetch fresh instead of serving nothing. + await entry.materializing + if (entry.materialized && Date.now() - entry.materialized.fetchedAt < maxAgeMs) return + } + const generationAtStart = entry.staleGeneration ?? 0 + const run = (async () => { + try { + const { deployed, draft, noDeployed } = await getDraftDiffValues( + entry.row.kind, + entry.row.path, + workspace + ) + let before = noDeployed ? undefined : deployed + let after: unknown = draft + const valueMasked = entry.row.kind === 'variable' + let valueUncomparable = false + if (valueMasked) { + ;({ before, after, valueUncomparable } = maskVariableDiffSides(before, after)) + } + const parts = computeDiffParts(before, after, 'deployed', 'draft') + // A save that landed mid-fetch invalidated this run's inputs. + if ((entry.staleGeneration ?? 0) !== generationAtStart) return + entry.materialized = { + status: noDeployed ? 'new' : parts.hasChanges ? 'modified' : 'unchanged', + patch: parts.patch, + lineCount: parts.lineCount, + files: parts.files, + valueMasked, + valueUncomparable, + noDeployed, + fetchedAt: Date.now() + } + } catch (e) { + if ((entry.staleGeneration ?? 0) !== generationAtStart) return + entry.materialized = { + status: 'error', + patch: '', + lineCount: 0, + noDeployed: false, + errorMessage: + (e as { status?: number } | null | undefined)?.status === 404 + ? 'item not found (the draft may reference a deleted item)' + : ((e as Error | null | undefined)?.message ?? 'failed to compute diff'), + fetchedAt: Date.now() + } + } + })() + entry.materializing = run + try { + await run + } finally { + entry.materializing = undefined + } +} + +function toView(entry: CacheEntry): WorkspaceDiffEntryView { + const m = entry.materialized + return { + kind: entry.row.kind, + type: entry.type, + triggerKind: entry.triggerKind, + path: entry.displayPath, + storagePath: entry.row.path, + summary: entry.row.summary, + status: m ? m.status : entry.type === undefined ? 'not_diffable' : 'pending', + patch: m?.patch, + patchLineCount: m?.lineCount, + files: m?.files, + valueMasked: m?.valueMasked, + valueUncomparable: m?.valueUncomparable, + noDeployed: m?.noDeployed, + errorMessage: m?.errorMessage + } +} + +/** Workspace index: every draft of the current user with its change status, + * materializing missing/stale patches up to the eager cap (`materializeAll` + * lifts the cap — search needs every patch). */ +export async function getWorkspaceDiffIndex( + workspace: string, + opts: { materializeAll?: boolean } = {} +): Promise { + const cache = await reconcile(workspace) + const addressable = [...cache.entries.values()].filter((e) => e.type !== undefined) + let toMaterialize = addressable.filter( + (e) => !e.materialized || Date.now() - e.materialized.fetchedAt >= INDEX_ENTRY_STALE_MS + ) + if (!opts.materializeAll) { + toMaterialize = toMaterialize.slice(0, EAGER_MATERIALIZE_CAP) + } + await mapPool(toMaterialize, FETCH_CONCURRENCY, (e) => + materialize(workspace, e, INDEX_ENTRY_STALE_MS) + ) + return { + entries: [...cache.entries.values()].map(toView), + otherUsersDraftCount: cache.otherUsersDraftCount + } +} + +/** Resolve a requested path to the draft row that owns it — by exact key or + * friendly draft_path — across the given kinds. Item mode flushes/probes the + * RESOLVED key: a renamed classic app's cell lives at its original storage + * path, which only the listing knows. */ +export async function resolveWorkspaceDiffTarget( + workspace: string, + kinds: UserDraftItemKind[], + path: string +): Promise<{ kind: UserDraftItemKind; storagePath: string } | undefined> { + const cache = await reconcile(workspace) + for (const kind of kinds) { + if (cache.entries.has(entryKey(kind, path))) return { kind, storagePath: path } + } + const entry = [...cache.entries.values()].find( + (e) => kinds.includes(e.row.kind) && (e.displayPath === path || e.row.path === path) + ) + return entry ? { kind: entry.row.kind, storagePath: entry.row.path } : undefined +} + +/** One item's diff entry, addressed by storage path or friendly draft path. + * Returns undefined when the current user has no draft there. */ +export async function readWorkspaceDiffEntry( + workspace: string, + itemKind: UserDraftItemKind, + path: string +): Promise { + const cache = await reconcile(workspace) + let entry = cache.entries.get(entryKey(itemKind, path)) + if (!entry) { + // Friendly-path addressing (draft-only items park at u/{user}/draft_{uuid}) + // and the classic-app/raw-app kind pair, which share the chat type 'app'. + const kinds: UserDraftItemKind[] = + itemKind === 'raw_app' || itemKind === 'app' ? ['raw_app', 'app'] : [itemKind] + entry = [...cache.entries.values()].find( + (e) => kinds.includes(e.row.kind) && (e.displayPath === path || e.row.path === path) + ) + } + if (!entry) return undefined + await materialize(workspace, entry, READ_ENTRY_REUSE_MS) + return toView(entry) +} + +// --------------------------------------------------------------------------- +// Fork mode: deployed fork vs deployed parent, mirroring the compare page. +// The index comes from the shared `compareWorkspaces` fetch (same tally the +// fork banner shows); per-item content is fetched with the same +// `getItemValue` canonicalization the compare page's diff drawer uses, so the +// patches match what merge-to-parent would actually ship. Local drafts are +// NOT part of the comparison — they are only flagged on the entries. +// --------------------------------------------------------------------------- + +/** How long a fetched comparison keeps serving fork index/read calls before a + * fresh tally is requested. In-app deploys bump the drafts version and force a + * refetch regardless. */ +const FORK_COMPARISON_REUSE_MS = 30_000 + +export type ForkDiffStatus = + | 'modified' + | 'only_in_fork' + | 'deleted_in_fork' + | 'unchanged' + | 'pending' + | 'error' + +export interface ForkDiffEntryView { + /** Comparison kind (per-kind trigger names, plus folder / resource_type). */ + kind: string + /** Chat-facing addressing; undefined when the chat cannot read the kind. */ + type?: WorkspaceItemType + triggerKind?: TriggerKind + path: string + ahead: number + behind: number + /** The current user also has a local draft on this item — not part of the + * deployed-vs-deployed comparison. */ + hasLocalDraft: boolean + status: ForkDiffStatus + patch?: string + patchLineCount?: number + /** Per-file patches for multi-file apps (changed files only). */ + files?: Record + /** A secret's content was placeholder-masked — content-only changes on + * this item cannot appear in the patch. */ + valueMasked?: boolean + errorMessage?: string +} + +export interface ForkDiffIndexView { + parentWorkspaceId: string + skippedComparison: boolean + entries: ForkDiffEntryView[] + hiddenAheadCount: number + hiddenBehindCount: number +} + +interface ForkMaterialized { + status: 'modified' | 'only_in_fork' | 'deleted_in_fork' | 'unchanged' | 'error' + patch: string + lineCount: number + files?: Record + valueMasked?: boolean + errorMessage?: string + fetchedAt: number +} + +interface ForkEntry { + kind: string + path: string + ahead: number + behind: number + existsInParent: boolean + existsInFork: boolean + type?: WorkspaceItemType + triggerKind?: TriggerKind + hasLocalDraft: boolean + materialized?: ForkMaterialized + materializing?: Promise +} + +interface ForkCache { + parentWorkspaceId: string + draftsVersion: number + parentDraftsVersion: number + /** Mutation epoch the producer started under — joiners and the reuse gate + * compare it against the current epoch, so a bump that landed before they + * arrived (not just mid-await) is still detected. */ + epoch: number + /** When the underlying comparison request STARTED (from its meta), not + * when this snapshot adopted it — aging from adoption time would let a + * near-expiry comparison live a whole extra reuse window here. */ + fetchedAt: number + /** Comparison-store generation; reuse stops the moment + * `invalidateWorkspaceComparison` fences it (e.g. a deploy whose draft + * cleanup failed and thus never bumped the drafts version). */ + comparisonGeneration: number + skippedComparison: boolean + entries: Map + hiddenAheadCount: number + hiddenBehindCount: number +} + +const forkCaches = new Map() +const forkReconciling = new Map>() + +/** Parent workspace id when `workspace` is a fork/dev workspace, else undefined. */ +export function getForkParentWorkspaceId(workspace: string): string | undefined { + return get(userWorkspaces).find((w) => w.id === workspace)?.parent_workspace_id ?? undefined +} + +const CHAT_TRIGGER_KINDS = new Set(TRIGGER_KINDS) + +function forkKindAddressing( + kind: string +): { type: WorkspaceItemType; triggerKind?: TriggerKind } | undefined { + switch (kind) { + case 'script': + case 'flow': + case 'resource': + case 'variable': + case 'schedule': + return { type: kind as WorkspaceItemType } + case 'app': + case 'raw_app': + return { type: 'app' } + default: { + const triggerKind = kind.endsWith('_trigger') ? kind.slice(0, -'_trigger'.length) : undefined + return triggerKind && CHAT_TRIGGER_KINDS.has(triggerKind) + ? { type: 'trigger', triggerKind: triggerKind as TriggerKind } + : undefined + } + } +} + +/** Draft kind holding local drafts for a comparison kind (for the flag join). */ +function draftKindForForkKind(kind: string): UserDraftItemKind | undefined { + if (kind === 'schedule') return 'trigger_schedule' + if (kind.endsWith('_trigger')) { + const t = kind.slice(0, -'_trigger'.length) + return `trigger_${t}` as UserDraftItemKind + } + if (kind === 'resource_type' || kind === 'folder') return undefined + return kind as UserDraftItemKind +} + +async function reconcileFork(workspace: string, parentWorkspaceId: string): Promise { + ensureCacheOwner() + // Same epoch/version fencing as `reconcile`: joiners re-validate after + // awaiting, producers refuse to store pre-mutation inputs (bounded retries). + for (let attempt = 0; ; attempt++) { + const prev = forkCaches.get(workspace) + const version = getWorkspaceDraftsVersion(workspace) + const parentVersion = getWorkspaceDraftsVersion(parentWorkspaceId) + const epoch = mutationEpoch(workspace) + const isCurrent = (c: ForkCache) => + c.parentWorkspaceId === parentWorkspaceId && + c.draftsVersion === getWorkspaceDraftsVersion(workspace) && + c.parentDraftsVersion === getWorkspaceDraftsVersion(parentWorkspaceId) + if ( + prev && + isCurrent(prev) && + prev.epoch === epoch && + prev.draftsVersion === version && + Date.now() - prev.fetchedAt < FORK_COMPARISON_REUSE_MS && + isComparisonCurrent(parentWorkspaceId, workspace, prev.comparisonGeneration) + ) { + return prev + } + const inflight = forkReconciling.get(workspace) + if (inflight) { + const joined = await inflight + if ((isCurrent(joined) && joined.epoch === mutationEpoch(workspace)) || attempt >= 4) { + return joined + } + continue + } + const run = (async () => { + // A drafts-version bump means something deployed in-app: demand a fresh + // tally. Otherwise piggyback on a recent fetch (e.g. the fork banner's). + const comparisonMaxAge = + prev && (prev.draftsVersion !== version || prev.parentDraftsVersion !== parentVersion) + ? 0 + : FORK_COMPARISON_REUSE_MS + const [comparisonMeta, draftsCache] = await Promise.all([ + fetchWorkspaceComparisonMeta(parentWorkspaceId, workspace, { maxAgeMs: comparisonMaxAge }), + reconcile(workspace) + ]) + const comparison = comparisonMeta.comparison + const localDraftKeys = new Set( + [...draftsCache.entries.values()].map((e) => entryKey(e.row.kind, e.row.path)) + ) + const entries = new Map() + for (const diff of comparison.diffs as WorkspaceItemDiff[]) { + const key = `${diff.kind}:${diff.path}` + const old = prev?.entries.get(key) + const reusable = + old && + prev!.draftsVersion === version && + old.ahead === diff.ahead && + old.behind === diff.behind && + old.existsInParent === diff.exists_in_source && + old.existsInFork === diff.exists_in_fork + ? old.materialized + : undefined + const addressing = forkKindAddressing(diff.kind) + const draftKind = draftKindForForkKind(diff.kind) + entries.set(key, { + kind: diff.kind, + path: diff.path, + ahead: diff.ahead, + behind: diff.behind, + existsInParent: diff.exists_in_source, + existsInFork: diff.exists_in_fork, + type: addressing?.type, + triggerKind: addressing?.triggerKind, + hasLocalDraft: draftKind ? localDraftKeys.has(entryKey(draftKind, diff.path)) : false, + materialized: reusable + }) + } + const cache: ForkCache = { + parentWorkspaceId, + draftsVersion: version, + parentDraftsVersion: parentVersion, + epoch, + fetchedAt: comparisonMeta.fetchedAt, + comparisonGeneration: comparisonMeta.generation, + skippedComparison: comparison.skipped_comparison, + entries, + hiddenAheadCount: comparison.hidden_ahead?.total ?? 0, + hiddenBehindCount: comparison.hidden_behind?.total ?? 0 + } + return cache + })() + forkReconciling.set(workspace, run) + let cache: ForkCache + try { + cache = await run + } finally { + if (forkReconciling.get(workspace) === run) forkReconciling.delete(workspace) + } + if ((isCurrent(cache) && mutationEpoch(workspace) === epoch) || attempt >= 4) { + forkCaches.set(workspace, cache) + return cache + } + // A deploy/save landed mid-fetch: this tally's inputs are pre-mutation. + } +} + +// App-row fields that differ between workspaces without being part of what a +// merge deploys (ids, version history, audit fields) — plus bundle_secret, +// which must never reach a tool result. +const APP_ROW_CROSS_WORKSPACE_IGNORE = new Set([ + 'id', + 'workspace_id', + 'versions', + 'created_by', + 'created_at', + 'extra_perms', + 'bundle_secret' +]) + +/** Inline flow scripts pin a `hash` the server recomputes per workspace — + * never comparable across workspaces. Generic deep walk so nested module + * shapes (loops, branches) need no taxonomy here. */ +function stripInlineFlowHashes(node: unknown): void { + if (Array.isArray(node)) { + for (const item of node) stripInlineFlowHashes(item) + return + } + if (node === null || typeof node !== 'object') return + const obj = node as Record + if (obj.value?.type === 'script' && obj.value.hash != undefined) { + obj.value.hash = undefined + } + for (const child of Object.values(obj)) stripInlineFlowHashes(child) +} + +interface ForkSideValue { + value: unknown + /** The variable's content was replaced by the placeholder — content-only + * changes on this item are invisible in the patch. */ + valueMasked: boolean +} + +async function fetchForkSideValue( + kind: string, + path: string, + workspace: string +): Promise { + // Variable VALUES never reach a tool result — the chat-wide invariant, not + // just for secrets (and secrets are additionally never decrypted). The + // placeholder is identical on both sides, so a value-only change is + // invisible here; the `valueMasked` flag lets callers say so instead of + // claiming "unchanged". + if (kind === 'variable') { + const variable = await VariableService.getVariable({ + workspace, + path, + decryptSecret: false + }) + return { + value: { + description: variable.description, + is_secret: variable.is_secret, + value: VARIABLE_VALUE_PLACEHOLDER + }, + valueMasked: true + } + } + // The shared getItemValue projection drops fields the backend comparison + // DOES consider (script/resource description, resource_type) — a diff of + // those fields alone would then falsely read "content matches parent". + // Project those kinds directly with the metadata included. + if (kind === 'script') { + const script = await ScriptService.getScriptByPath({ workspace, path }) + return { + value: { + content: script.content, + lock: script.lock, + schema: script.schema, + summary: script.summary, + description: script.description, + language: script.language + }, + valueMasked: false + } + } + if (kind === 'resource') { + const resource = await ResourceService.getResource({ workspace, path }) + return { + value: { + value: resource.value, + description: resource.description, + resource_type: resource.resource_type + }, + valueMasked: false + } + } + if (kind === 'flow') { + // The shared projection drops `schema`, which the backend comparison + // counts — a schema-only change would falsely read "matches parent". + const flow = await FlowService.getFlowByPath({ workspace, path }) + const value = structuredClone(flow.value) + stripInlineFlowHashes(value) + return { + value: { + summary: flow.summary, + description: flow.description, + schema: flow.schema, + value + }, + valueMasked: false + } + } + if (kind === 'resource_type') { + const rt = await ResourceService.getResourceType({ workspace, path }) + return { + value: { + schema: rt.schema, + description: rt.description, + format_extension: rt.format_extension, + is_fileset: rt.is_fileset + }, + valueMasked: false + } + } + const value = await getItemValue(kind as DeployKind, path, workspace) + // getItemValue reads `{}` for ANY failed fetch ("item may not exist") — but + // a fork side is only fetched when the comparison lists it as existing, so + // an empty read is a transient failure. Erroring (surfaced as a fetch-error + // entry) beats fabricating a one-sided or matching diff out of it. + if (value == null || (typeof value === 'object' && Object.keys(value).length === 0)) { + throw new Error(`failed to read ${kind} ${path} in ${workspace}`) + } + if ((kind === 'app' || kind === 'raw_app') && value !== null && typeof value === 'object') { + const row = value as Record + // Raw apps: project onto the flat files/runnables draft shape so per-file + // splitting works and the sides match the draft-mode canonicalization. + // parent_version is a per-workspace version counter — never comparable + // across workspaces. Inline-script locks are server-recomputed noise. + if (kind === 'raw_app' || row.raw_app === true) { + const canonical = appSourceToDraftValue(row) as Record + delete canonical.parent_version + const runnables = canonical.runnables as Record | undefined + if (runnables) { + for (const k of Object.keys(runnables)) { + if (runnables[k]?.inlineScript?.lock != undefined) { + runnables[k].inlineScript.lock = undefined + } + } + } + return { value: canonical, valueMasked: false } + } + return { + value: Object.fromEntries( + Object.entries(row).filter(([k]) => !APP_ROW_CROSS_WORKSPACE_IGNORE.has(k)) + ), + valueMasked: false + } + } + return { value, valueMasked: false } +} + +async function materializeFork( + workspace: string, + parentWorkspaceId: string, + entry: ForkEntry, + maxAgeMs: number +): Promise { + if (entry.materialized && Date.now() - entry.materialized.fetchedAt < maxAgeMs) return + if (entry.materializing) return entry.materializing + const run = (async () => { + try { + const [parentSide, forkSide] = await Promise.all([ + entry.existsInParent + ? fetchForkSideValue(entry.kind, entry.path, parentWorkspaceId) + : undefined, + entry.existsInFork ? fetchForkSideValue(entry.kind, entry.path, workspace) : undefined + ]) + const parentValue = parentSide?.value + const forkValue = forkSide?.value + const valueMasked = parentSide?.valueMasked === true || forkSide?.valueMasked === true + const oneSidedStatus = !entry.existsInFork + ? 'deleted_in_fork' + : !entry.existsInParent + ? 'only_in_fork' + : undefined + const parts = computeDiffParts(parentValue, forkValue, 'parent', 'fork') + entry.materialized = { + status: oneSidedStatus ?? (parts.hasChanges ? 'modified' : 'unchanged'), + patch: parts.patch, + lineCount: parts.lineCount, + files: parts.files, + valueMasked, + fetchedAt: Date.now() + } + } catch (e) { + entry.materialized = { + status: 'error', + patch: '', + lineCount: 0, + errorMessage: (e as Error | null | undefined)?.message ?? 'failed to compute diff', + fetchedAt: Date.now() + } + } + })() + entry.materializing = run + try { + await run + } finally { + entry.materializing = undefined + } +} + +function toForkView(entry: ForkEntry): ForkDiffEntryView { + const m = entry.materialized + return { + kind: entry.kind, + type: entry.type, + triggerKind: entry.triggerKind, + path: entry.path, + ahead: entry.ahead, + behind: entry.behind, + hasLocalDraft: entry.hasLocalDraft, + status: m?.status ?? 'pending', + patch: m?.patch, + patchLineCount: m?.lineCount, + files: m?.files, + valueMasked: m?.valueMasked, + errorMessage: m?.errorMessage + } +} + +/** Cheap comparison metadata (no patch materialization) — lets item/search + * modes distinguish "no differences" from "comparison unavailable". */ +export async function getForkComparisonStatus( + workspace: string, + parentWorkspaceId: string +): Promise<{ skippedComparison: boolean }> { + const cache = await reconcileFork(workspace, parentWorkspaceId) + return { skippedComparison: cache.skippedComparison } +} + +/** Fork index: every item that differs between the fork and its parent + * (`materializeAll` lifts the eager cap — search needs every patch). */ +export async function getForkDiffIndex( + workspace: string, + parentWorkspaceId: string, + opts: { materializeAll?: boolean } = {} +): Promise { + const cache = await reconcileFork(workspace, parentWorkspaceId) + let toMaterialize = [...cache.entries.values()].filter( + (e) => !e.materialized || Date.now() - e.materialized.fetchedAt >= INDEX_ENTRY_STALE_MS + ) + if (!opts.materializeAll) { + toMaterialize = toMaterialize.slice(0, EAGER_MATERIALIZE_CAP) + } + await mapPool(toMaterialize, FETCH_CONCURRENCY, (e) => + materializeFork(workspace, parentWorkspaceId, e, INDEX_ENTRY_STALE_MS) + ) + return { + parentWorkspaceId: cache.parentWorkspaceId, + skippedComparison: cache.skippedComparison, + entries: [...cache.entries.values()].map(toForkView), + hiddenAheadCount: cache.hiddenAheadCount, + hiddenBehindCount: cache.hiddenBehindCount + } +} + +/** Fork-vs-parent entries for one path. `kinds` lists the comparison kinds + * the chat type maps to (e.g. type 'app' → ['app', 'raw_app']); an EMPTY list + * is a path-only wildcard — how kinds outside the chat type enum (folder, + * resource_type, …) stay readable. A wildcard returns EVERY kind differing at + * the path (nothing in the chat schema could disambiguate them); typed reads + * return at most one. Empty array = no difference at that path. */ +export async function readForkDiffEntries( + workspace: string, + parentWorkspaceId: string, + kinds: string[], + path: string +): Promise { + const cache = await reconcileFork(workspace, parentWorkspaceId) + let entries: ForkEntry[] = [] + if (kinds.length === 0) { + entries = [...cache.entries.values()].filter((e) => e.path === path) + } else { + for (const kind of kinds) { + const entry = cache.entries.get(`${kind}:${path}`) + if (entry) { + entries = [entry] + break + } + } + } + for (const entry of entries) { + await materializeFork(workspace, parentWorkspaceId, entry, READ_ENTRY_REUSE_MS) + } + return entries.map(toForkView) +} diff --git a/frontend/src/lib/components/copilot/chat/global/draftDiff.test.ts b/frontend/src/lib/components/copilot/chat/global/draftDiff.test.ts new file mode 100644 index 0000000000..dac4df08f5 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/draftDiff.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from 'vitest' +import { changedLineIndices, draftDeployedPatch, textFilePatch } from './draftDiff' + +describe('draftDeployedPatch', () => { + it('returns an empty string for identical values', () => { + expect(draftDeployedPatch({ a: 1, b: 'x' }, { a: 1, b: 'x' })).toBe('') + }) + + it('ignores key-order differences', () => { + expect(draftDeployedPatch({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe('') + }) + + it('treats null fields and absent fields as equal', () => { + expect(draftDeployedPatch({ a: 1, tag: null }, { a: 1 })).toBe('') + // but a real value vs null/absent still diffs + expect(draftDeployedPatch({ a: 1, tag: 'prod' }, { a: 1 })).toContain('-tag: prod') + }) + + it('diffs multiline code fields line-by-line', () => { + const deployed = { content: 'line1\nline2\nline3\n', language: 'bun' } + const draft = { content: 'line1\nchanged\nline3\n', language: 'bun' } + const patch = draftDeployedPatch(deployed, draft) + expect(patch).toContain('- line2') + expect(patch).toContain('+ changed') + // Unchanged lines are context, not part of the change + expect(patch).not.toContain('- line1') + }) + + it('renders a whole draft as additions when there is no deployed side', () => { + const patch = draftDeployedPatch(undefined, { summary: 'new item', value: { modules: [] } }) + expect(patch).toContain('+summary: new item') + expect(patch).not.toMatch(/^-[^-]/m) + }) +}) + +describe('windowPatch', () => { + it('continues from the last delivered line when the char cap cuts a window', async () => { + const { windowPatch } = await import('./draftDiff') + const patch = Array.from({ length: 20 }, (_, i) => `line-${String(i).padStart(2, '0')}`).join( + '\n' + ) + // Cap fits ~5 complete 7-char lines ("line-NN" + newline). + const out = windowPatch(patch, 0, 20, 40) + const continueAt = Number(out.match(/offset=(\d+)/)?.[1]) + expect(continueAt).toBeGreaterThan(0) + expect(continueAt).toBeLessThan(20) + // The continuation must resume exactly at the first undelivered line. + const delivered = out.split('\n').filter((l) => l.startsWith('line-')) + expect(delivered[delivered.length - 1]).toBe(`line-${String(continueAt - 1).padStart(2, '0')}`) + const next = windowPatch(patch, continueAt, 20, 40) + expect(next).toContain(`line-${String(continueAt).padStart(2, '0')}`) + }) + + it('reports an overshooting offset instead of an impossible range', async () => { + const { windowPatch } = await import('./draftDiff') + expect(windowPatch('a\nb', 10, 5, 100)).toContain('no lines at offset 10') + }) + + it('steps past a single line larger than the whole budget', async () => { + const { windowPatch } = await import('./draftDiff') + const patch = ['x'.repeat(100), 'after'].join('\n') + const out = windowPatch(patch, 0, 10, 40) + expect(out).toContain('offset=1') + const next = windowPatch(patch, 1, 10, 40) + expect(next).toContain('after') + }) +}) + +describe('changedLineIndices', () => { + it('counts source lines starting with ++ or -- but never the file labels', () => { + const patch = textFilePatch('a\n--counter\nb\n', 'a\n++counter\nb\n', 'deployed', 'draft') + const lines = patch.split('\n') + const changed = changedLineIndices(patch).map((i) => lines[i]) + // A changed `--counter`/`++counter` source line is byte-identical to a + // file label prefix — only hunk-awareness keeps it. + expect(changed).toContain('---counter') + expect(changed).toContain('+++counter') + // The actual file labels never count as changes. + expect(changed.some((l) => l.includes('deployed') || l.includes('draft'))).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/draftDiff.ts b/frontend/src/lib/components/copilot/chat/global/draftDiff.ts new file mode 100644 index 0000000000..0fb12d480c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/draftDiff.ts @@ -0,0 +1,132 @@ +import { createTwoFilesPatch } from 'diff' +import YAML from 'yaml' + +// Drop null/undefined object properties so `field: null` on one side never +// diffs against the field being absent on the other (backend rows spell unset +// as null, draft payloads omit the key). A real change keeps showing: the side +// that has a value still emits its line. Array elements are kept (recursed +// into, not removed) so positions stay aligned. +function pruneNulls(value: unknown): unknown { + if (Array.isArray(value)) return value.map(pruneNulls) + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, v]) => v != null) + .map(([k, v]) => [k, pruneNulls(v)]) + ) + } + return value +} + +// Serialize a draft/deployed diff value to deterministic YAML: sorted keys so +// field order never shows as a change, no anchors/aliases (a `*a` reference in +// a diff is unreadable), no line folding (wrapping would split one logical +// change across hunks). Multiline strings (script content, app files) render +// as block scalars, so code diffs read line-by-line instead of as one escaped +// JSON string. +export function toStableYaml(value: unknown): string { + if (value == null) return '' + return ( + YAML.stringify(pruneNulls(value), { + sortMapEntries: true, + aliasDuplicateObjects: false, + lineWidth: 0 + }) ?? '' + ) +} + +/** Unified patch between two sides of a workspace item. + * Returns '' when the two sides are identical. */ +export function yamlValuePatch( + before: unknown, + after: unknown, + beforeLabel: string, + afterLabel: string +): string { + const beforeYaml = toStableYaml(before) + const afterYaml = toStableYaml(after) + if (beforeYaml === afterYaml) return '' + return createTwoFilesPatch(beforeLabel, afterLabel, beforeYaml, afterYaml, '', '', { context: 3 }) +} + +/** Unified patch between the deployed and draft sides of a workspace item. + * Returns '' when the two sides are identical. */ +export function draftDeployedPatch(deployed: unknown, draft: unknown): string { + return yamlValuePatch(deployed, draft, 'deployed', 'draft') +} + +/** Indices of the real changed lines (+/-) in a unified patch. Hunk-aware: + * the `---`/`+++` file-label lines are structure, but they only occur before + * the first `@@` marker — a changed SOURCE line may itself start with ++ or + * -- (e.g. `++counter`), so prefix-matching `+++`/`---` would drop it. */ +export function changedLineIndices(patch: string): number[] { + const indices: number[] = [] + const lines = patch.split('\n') + let inHunk = false + for (let i = 0; i < lines.length; i++) { + if (lines[i].startsWith('@@')) { + inHunk = true + continue + } + if (!inHunk) continue + if (lines[i].startsWith('+') || lines[i].startsWith('-')) indices.push(i) + } + return indices +} + +/** Unified patch between two raw text files (no YAML wrapping — file contents + * diff line-by-line as-is). Returns '' when identical; an absent side is + * treated as empty, so a one-sided file reads as all additions/removals. */ +export function textFilePatch( + before: string | undefined, + after: string | undefined, + beforeLabel: string, + afterLabel: string +): string { + const beforeText = before ?? '' + const afterText = after ?? '' + if (beforeText === afterText) return '' + return createTwoFilesPatch(beforeLabel, afterLabel, beforeText, afterText, '', '', { context: 3 }) +} + +/** Window a patch by lines with a character backstop. When the backstop cuts + * inside the window, the continuation offset advances only past the COMPLETE + * lines actually delivered — a pre-truncation offset would skip the rest of + * the window forever. */ +export function windowPatch( + patch: string, + offset: number, + limit: number, + maxChars: number +): string { + const lines = patch.split('\n') + const total = lines.length + if (offset >= total) { + return `(no lines at offset ${offset} — the patch has ${total} lines; call again with a smaller offset)` + } + const start = offset + const requestedEnd = Math.min(total, start + limit) + let body = lines.slice(start, requestedEnd).join('\n') + let effectiveEnd = requestedEnd + let note = '' + if (body.length > maxChars) { + body = body.slice(0, maxChars) + const lastNewline = body.lastIndexOf('\n') + if (lastNewline === -1) { + // One pathological line exceeds the budget: deliver its head and step + // past it, or pagination could never advance. + effectiveEnd = start + 1 + note = `\n… [line truncated at ${maxChars} chars]` + } else { + body = body.slice(0, lastNewline) + effectiveEnd = start + body.split('\n').length + note = `\n… window truncated at ${maxChars} chars.` + } + } + if (start > 0 || effectiveEnd < total) { + note += `\n(lines ${start + 1}-${effectiveEnd} of ${total}${ + effectiveEnd < total ? `; call diff again with offset=${effectiveEnd} for the rest` : '' + })` + } + return body + note +} diff --git a/frontend/src/lib/components/copilot/chat/global/gate.ts b/frontend/src/lib/components/copilot/chat/global/gate.ts index df8035de87..f3eb731ae2 100644 --- a/frontend/src/lib/components/copilot/chat/global/gate.ts +++ b/frontend/src/lib/components/copilot/chat/global/gate.ts @@ -1,26 +1,53 @@ /** - * Dev-only gate for the Global AI chat mode. + * Beta opt-out gate for AI Sessions (and the Global AI chat mode). * - * While the mode is being iterated on, it should not be visible to regular - * users. Developers/QA enable it in their browser with: + * Sessions ship enabled by default. Users can switch back to the legacy + * docked chat from the beta banner under the session chat, which stores an + * opt-out in this browser's localStorage; the mirror banner in the legacy + * chat switches back. Both toggles do a full page reload: every call site + * reads the gate once at init, so a live flip would leave the UI half-switched. * - * localStorage.setItem('wm_dev_global_ai', '1') - * - * and reload the page. To disable, remove the key or set it to anything else. - * - * When the mode is ready to ship to everyone, replace every call to - * `isGlobalAiEnabled()` with `true` and delete this file. The references are - * intentionally narrow (chat mode visibility, custom prompt settings, the - * `change_mode` tool enum, the AI skills workspace settings tab, and the - * `/global_drafts` dev route) so the rip-out is a small grep. + * When the beta ends, replace every call to `isGlobalAiEnabled()` with `true` + * and delete this file. The references are intentionally narrow (chat mode + * visibility, custom prompt settings, the `change_mode` tool enum, and the + * AI skills workspace settings tab) so the rip-out is a small grep. */ -const STORAGE_KEY = 'wm_dev_global_ai' +import { logFeatureUsage } from '$lib/utils/featureUsage' + +const OPT_OUT_KEY = 'wm_sessions_beta_optout' export function isGlobalAiEnabled(): boolean { if (typeof localStorage === 'undefined') return false try { - return localStorage.getItem(STORAGE_KEY) === '1' + return localStorage.getItem(OPT_OUT_KEY) !== '1' } catch { return false } } + +/** Persist the opt-out choice, then hard-reload so every gated site re-reads it. */ +export function setSessionsBetaOptOut(optOut: boolean, target: string) { + // Navigate even when persistence throws (quota, private browsing) — the + // button must not be a silent no-op. The reload then shows the unchanged + // mode, which is the honest feedback that the toggle didn't stick. + let persisted = true + try { + if (optOut) { + localStorage.setItem(OPT_OUT_KEY, '1') + // Land with the legacy pane open — on a fresh profile the pane + // defaults to closed, and the round-trip must end in a visible chat + // (with its reactivation banner), not on a bare workspace page. + localStorage.setItem('ai-chat-open', 'true') + } else { + localStorage.removeItem(OPT_OUT_KEY) + } + } catch { + persisted = false + } + // Anonymous usage counter on the shared feature_usage channel. The buffer's + // pagehide flush + keepalive fetch carry it across the hard navigation below. + if (persisted) { + logFeatureUsage('ai_session', optOut ? 'beta_optout' : 'beta_optin') + } + window.location.href = target +} diff --git a/frontend/src/lib/components/copilot/chat/global/pageNavigation.test.ts b/frontend/src/lib/components/copilot/chat/global/pageNavigation.test.ts index 5f7a84ee1c..3fc0eb0b69 100644 --- a/frontend/src/lib/components/copilot/chat/global/pageNavigation.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/pageNavigation.test.ts @@ -5,8 +5,10 @@ import { buildResourcesUrl, buildVariablesUrl, buildTriggersUrl, - buildFoldersUrl + buildFoldersUrl, + buildCompareUrl } from './pageNavigation' +import { parseItemsMaskParam } from '$lib/components/sessions/modifiedItemsMask' function parse(appPath: string): URL { return new URL(appPath, 'http://x') @@ -37,20 +39,34 @@ describe('pageNavigation builders', () => { it('resources keeps resource_type + path, drops runs-only keys', () => { const u = parse( - buildResourcesUrl({ resource_type: 'postgres', path: 'f/x', status: 'failure' }) + buildResourcesUrl({ filters: { resource_type: 'postgres', path: 'f/x', status: 'failure' } }) ) expect(u.searchParams.get('resource_type')).toBe('postgres') expect(u.searchParams.get('path')).toBe('f/x') expect(u.searchParams.has('status')).toBe(false) }) + it('resources opens a specific resource via the #/resource/ hash', () => { + const u = parse(buildResourcesUrl({ open: 'f/x/db' })) + expect(u.pathname).toBe('/resources') + expect(u.hash).toBe('#/resource/f/x/db') + }) + it('variables keeps path + owner only', () => { - const u = parse(buildVariablesUrl({ path: 'f/x', owner: 'u/alice', resource_type: 'postgres' })) + const u = parse( + buildVariablesUrl({ filters: { path: 'f/x', owner: 'u/alice', resource_type: 'postgres' } }) + ) expect(u.searchParams.get('path')).toBe('f/x') expect(u.searchParams.get('owner')).toBe('u/alice') expect(u.searchParams.has('resource_type')).toBe(false) }) + it('variables opens a specific variable via hash', () => { + const u = parse(buildVariablesUrl({ open: 'u/alice/token' })) + expect(u.pathname).toBe('/variables') + expect(u.hash).toBe('#u/alice/token') + }) + it('triggers route to the kind page, opening a specific trigger via hash', () => { expect(parse(buildTriggersUrl({ trigger_kind: 'kafka' })).pathname).toBe('/kafka_triggers') const u = parse(buildTriggersUrl({ trigger_kind: 'http', open: 'f/a/b' })) @@ -63,4 +79,19 @@ describe('pageNavigation builders', () => { expect(u.pathname).toBe('/folders') expect(u.search).toBe('') }) + + it('compare carries workspace, mode, and an items mask that round-trips through the page parser', () => { + const items = ['script:f/foo/bar', 'trigger_schedule:u/alice/daily'] + const u = parse(buildCompareUrl({ workspace_id: 'wm-fork-x', mode: 'fork', items })) + expect(u.pathname).toBe('/forks/compare') + expect(u.searchParams.get('workspace_id')).toBe('wm-fork-x') + expect(u.searchParams.get('mode')).toBe('fork') + expect(parseItemsMaskParam(u.searchParams.get('items')!)).toEqual(new Set(items)) + }) + + it('compare omits mode and items when not provided', () => { + const u = parse(buildCompareUrl({ workspace_id: 'ws' })) + expect(u.searchParams.get('mode')).toBeNull() + expect(u.searchParams.get('items')).toBeNull() + }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/pageNavigation.ts b/frontend/src/lib/components/copilot/chat/global/pageNavigation.ts index ea783e19e7..cc5f091f9b 100644 --- a/frontend/src/lib/components/copilot/chat/global/pageNavigation.ts +++ b/frontend/src/lib/components/copilot/chat/global/pageNavigation.ts @@ -1,7 +1,15 @@ import { buildFilterUrl } from '$lib/navigation' import { buildRunsFilterSearchbarSchema } from '$lib/components/runs/runsFilter' import { buildSchedulesFilterSchema } from '$lib/components/schedules/schedulesFilter' -import { TRIGGER_PAGES, type TriggerKind } from '$lib/components/sessions/previewRouter' +import { + COMPARE_PAGE, + TRIGGER_PAGES, + type TriggerKind +} from '$lib/components/sessions/previewRouter' +import { + COMPARE_ITEMS_PARAM, + serializeItemsMaskParam +} from '$lib/components/sessions/modifiedItemsMask' // In-app paths for the deep-linkable preview pages the AI chat can open. export const RUNS_PATH = '/runs' @@ -84,13 +92,33 @@ export function buildSchedulesUrl({ // full filter schema), so the allow-list is the exact set of keys the builder emits — // these names match the query params the pages read (variablesFilter/resourcesFilter/ // assetsFilter and audit_logs/+page.svelte). -export function buildVariablesUrl(filters: Record): string { - return buildFilterUrl(VARIABLES_PATH, filters, { validKeys: ['path', 'owner'] }) +/** When `open` is set, the variable at that exact path is opened in the edit + * drawer via the `#` hash the page already handles. */ +export function buildVariablesUrl({ + open, + filters +}: { + open?: string + filters?: Record +}): string { + return buildFilterUrl(VARIABLES_PATH, filters ?? {}, { + validKeys: ['path', 'owner'], + hash: open + }) } -export function buildResourcesUrl(filters: Record): string { - return buildFilterUrl(RESOURCES_PATH, filters, { - validKeys: ['path', 'resource_type', 'owner'] +/** When `open` is set, the resource at that exact path is opened in the edit + * drawer via the `#/resource/` hash the page already handles. */ +export function buildResourcesUrl({ + open, + filters +}: { + open?: string + filters?: Record +}): string { + return buildFilterUrl(RESOURCES_PATH, filters ?? {}, { + validKeys: ['path', 'resource_type', 'owner'], + hash: open ? `/resource/${open}` : undefined }) } @@ -118,6 +146,36 @@ export function buildGroupsUrl(): string { return GROUPS_PATH } +/** + * Deep-link to the Compare & Deploy page (`/forks/compare`). `workspace_id` is required: + * inside a session preview the page loads with the *navigation* workspace as its store + * default, which is not necessarily the session's (possibly forked) workspace. `items` + * preselects exactly those `kind:path` entries (see modifiedItemsMask.ts); omitted, the + * page falls back to its select-all default. `mode` forces the draft or fork comparison; + * omitted, the page auto-picks: on a fork it lands on the view containing the masked + * items (draft when any of them is a pending draft, else the fork comparison); a + * non-fork always gets the draft view. + */ +export function buildCompareUrl({ + workspace_id, + mode, + items +}: { + workspace_id: string + mode?: 'draft' | 'fork' + items?: readonly string[] +}): string { + return buildFilterUrl( + COMPARE_PAGE.path, + { + workspace_id, + mode, + [COMPARE_ITEMS_PARAM]: items ? serializeItemsMaskParam(items) : undefined + }, + { validKeys: ['workspace_id', 'mode', COMPARE_ITEMS_PARAM] } + ) +} + /** * Deep-link to a trigger list page (by kind). When `open` is set, the trigger at that * exact path is opened in the edit drawer via the `#` hash the page handles. diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 342717f26e..59a66e4926 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -3,6 +3,7 @@ import { DraftService } from '$lib/gen' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { DEFAULT_DATA as DEFAULT_RAW_APP_DATA } from '$lib/components/raw_apps/dataTableRefUtils' import { UserDraft, type UserDraftEntry, type UserDraftItemKind } from '$lib/userDraft.svelte' +import { invalidateWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' import { getWorkspaceItemKey, type AppDraftValue, @@ -21,9 +22,11 @@ const TRIGGER_DRAFT_KIND_BY_TRIGGER_KIND = { nats: 'trigger_nats', postgres: 'trigger_postgres', mqtt: 'trigger_mqtt', + amqp: 'trigger_amqp', sqs: 'trigger_sqs', gcp: 'trigger_gcp', - azure: 'trigger_azure' + azure: 'trigger_azure', + email: 'trigger_email' } as const satisfies Record const TRIGGER_KIND_BY_DRAFT_KIND = Object.fromEntries( @@ -44,9 +47,11 @@ const GLOBAL_DRAFT_KINDS = [ 'trigger_nats', 'trigger_postgres', 'trigger_mqtt', + 'trigger_amqp', 'trigger_sqs', 'trigger_gcp', 'trigger_azure', + 'trigger_email', 'resource', 'variable' ] as const satisfies UserDraftItemKind[] @@ -67,7 +72,10 @@ function normalizeAppDraftValue(value: AppDraftValue): AppDraftValue { custom_path: value.custom_path, // Carry the fork-base version through the whitelist — it is dropped on every // save otherwise, which would defeat the stale-draft check. - parent_version: value.parent_version + parent_version: value.parent_version, + // Same for the friendly path of a draft-only app: dropping it here would + // rename the app back to its `draft_` storage key on every chat edit. + draft_path: value.draft_path } } @@ -129,10 +137,40 @@ export function triggerKindToUserDraftKind(kind: TriggerKind): UserDraftItemKind return TRIGGER_DRAFT_KIND_BY_TRIGGER_KIND[kind] } +/** Inverse of `itemKindFor`: the chat-facing type (+ trigger kind) for a draft + * kind. Classic and raw app drafts both surface as the chat's `app` type — + * mirroring the read path, which pairs the two kinds. `undefined` for kinds + * the chat cannot address (webhook / poll / cli trigger drafts, data + * pipelines). */ +export function itemTypeForKind( + kind: UserDraftItemKind +): { type: WorkspaceItemType; triggerKind?: TriggerKind } | undefined { + switch (kind) { + case 'script': + case 'flow': + case 'resource': + case 'variable': + return { type: kind } + case 'app': + case 'raw_app': + return { type: 'app' } + case 'trigger_schedule': + return { type: 'schedule' } + default: { + const triggerKind = TRIGGER_KIND_BY_DRAFT_KIND[kind] + return triggerKind ? { type: 'trigger', triggerKind } : undefined + } + } +} + function scriptDraftToWorkspaceItem(path: string, draft: NewScript): WorkspaceItem { return { type: 'script', path, + // The session editor parks a rename in the draft's `draft_path` (see + // sessionDraftCodecs.ts); surface it so lists/pickers show the friendly + // name instead of the `draft_` storage key. + draftPath: (draft as NewScript & { draft_path?: string }).draft_path, summary: draft.summary, language: draft.language, value: draft.content, @@ -145,6 +183,7 @@ function flowDraftToWorkspaceItem(path: string, draft: Flow): WorkspaceItem { return { type: 'flow', path, + draftPath: (draft as Flow & { draft_path?: string }).draft_path, summary: draft.summary, // The persisted flow draft carries `version_id` (the deployed head it was // forked from, pinned at fork by writeDraft/the editor) — the flow analog @@ -165,6 +204,7 @@ function appDraftToWorkspaceItem(path: string, draft: AppDraftValue): WorkspaceI return { type: 'app', path, + draftPath: value.draft_path, summary: value.summary, parentVersionId: value.parent_version, value, @@ -266,7 +306,14 @@ function userDraftEntryToWorkspaceItem( : undefined } } - return item && isLiveDraft ? { ...item, isLiveDraft: true } : item + if (!item) return undefined + // Drop a draftPath that just repeats `path` (no extra display information), + // keep it otherwise — including on live entries: a live editor that + // registers its storage key as the effective path (flow/raw-app renames + // live in the value's `draft_path`, not `path`) must not hide the staged + // rename from lists and pickers. + const draftPath = item.draftPath === item.path ? undefined : item.draftPath + return isLiveDraft ? { ...item, draftPath, isLiveDraft: true } : { ...item, draftPath } } function liveDisplayPath( @@ -438,9 +485,17 @@ export async function persistGlobalDraft( const conflict = opts.force ? undefined : UserDraftDbSyncer.getConflict({ workspace, itemKind, path: storagePath }).conflict - return conflict - ? { status: 'conflict', item, itemKind, storagePath, serverTimestamp: conflict.serverTimestamp } - : { status: 'saved', item, itemKind, storagePath } + if (conflict) { + return { + status: 'conflict', + item, + itemKind, + storagePath, + serverTimestamp: conflict.serverTimestamp + } + } + invalidateWorkspaceDrafts(workspace) + return { status: 'saved', item, itemKind, storagePath } } export async function getGlobalDraft( @@ -473,6 +528,7 @@ function backendDraftRowToWorkspaceItem( kind: string path: string summary?: string + draft_path?: string } ): WorkspaceItem | undefined { if (!(GLOBAL_DRAFT_KINDS as readonly string[]).includes(row.kind)) return undefined @@ -506,6 +562,12 @@ function backendDraftRowToWorkspaceItem( return { type, path: displayPath, + // The row's friendly path (from the draft JSON) names the item; only a + // draft_path that repeats the display path adds nothing. Kept even for + // live rows — a live registration whose effective path is the storage key + // (flow/raw-app renames live in the value's `draft_path`, not `path`) + // must not hide the staged rename. + draftPath: row.draft_path === displayPath ? undefined : row.draft_path, summary: row.summary, value: undefined, isDraft: true, @@ -584,6 +646,69 @@ export async function deleteGlobalDraft( ) } if (type === 'variable') clearEphemeralSecretVariableDraftValue(workspace, storagePath) + invalidateWorkspaceDrafts(workspace) +} + +/** Kind-addressed live-editor storage resolution (friendly → storage path), + * for callers that must probe several draft kinds per chat type. */ +export function resolveGlobalDraftStoragePathByKind( + workspace: string, + itemKind: UserDraftItemKind, + path: string +): string { + return resolveDraftStoragePath(workspace, itemKind, path) +} + +/** Local in-memory draft cell, kind-addressed: the chat `app` type spans two + * draft kinds (raw_app + classic app), so callers probing both address by + * kind. No backend fallback — this is the freshest state when a save is + * parked (auto-save off), failed, or conflicted; read-only callers use it + * instead of persisting. */ +export function readLocalDraftCellByKind( + workspace: string, + itemKind: UserDraftItemKind, + path: string +): unknown | undefined { + const storagePath = resolveDraftStoragePath(workspace, itemKind, path) + return UserDraft.get(itemKind, storagePath, { workspace }) +} + +/** Flush every parked local draft autosave for the workspace so the server + * listing reflects the latest edits — a brand-new editor draft has no server + * row until its first flush. No-ops per key when nothing is pending. + * Honors the auto-save toggle: with auto-save off, parked editor edits stay + * parked — a read-only caller must not persist what the user chose not to. + * `unflushedPaths` lists items whose latest edits did NOT reach the server + * (toggle-parked or failed save), so callers can say the listing excludes them. */ +export async function flushGlobalDraftSaves( + workspace: string +): Promise<{ unflushedPaths: string[] }> { + // Classic-app editor cells live under the `app` kind, which is deliberately + // NOT in GLOBAL_DRAFT_KINDS (clearGlobalDrafts must never clear a user's + // open classic editor) — but their unflushed edits must be flushed/reported + // like every other kind. + const drafts = UserDraft.list({ workspace, itemKinds: [...GLOBAL_DRAFT_KINDS, 'app'] }) + await Promise.all( + drafts.map((draft) => + UserDraftDbSyncer.flush( + { workspace, itemKind: draft.itemKind, path: draft.path }, + { honorAutosaveToggle: true } + ) + ) + ) + const unflushedPaths = drafts + .filter((draft) => { + const query = { workspace, itemKind: draft.itemKind, path: draft.path } + // A conflicted save also leaves the server without the local edits: + // the payload stays parked but the state is neither pending nor failed. + return ( + UserDraftDbSyncer.hasUnsavedDisabledChanges(query) || + UserDraftDbSyncer.getState(query).state === 'failed' || + UserDraftDbSyncer.getConflict(query).conflict !== undefined + ) + }) + .map((draft) => draft.path) + return { unflushedPaths } } export function clearGlobalDrafts(workspace: string): void { diff --git a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts index a8196a5054..529c87d493 100644 --- a/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts +++ b/frontend/src/lib/components/copilot/chat/global/workspaceItems.ts @@ -4,9 +4,11 @@ import type { CreateVariable, FlowValue, GcpTriggerData, + NewEmailTrigger, NewHttpTrigger, NewKafkaTrigger, NewMqttTrigger, + NewAmqpTrigger, NewNatsTrigger, NewPostgresTrigger, NewSchedule, @@ -38,9 +40,11 @@ export const TRIGGER_KINDS = [ 'nats', 'postgres', 'mqtt', + 'amqp', 'sqs', 'gcp', - 'azure' + 'azure', + 'email' ] as const export type TriggerKind = (typeof TRIGGER_KINDS)[number] @@ -52,9 +56,11 @@ export type TriggerRequestBody = | NewNatsTrigger | NewPostgresTrigger | NewMqttTrigger + | NewAmqpTrigger | NewSqsTrigger | GcpTriggerData | AzureTriggerData + | NewEmailTrigger export type WorkspaceItemType = | 'script' @@ -75,6 +81,10 @@ export type AppDraftValue = { // Fork base: the deployed app version this draft was started from, pinned at // fork. The app analog of a script's parent_hash / a flow's version_id. parent_version?: number + // User-typed friendly path while the app is parked at a `…/draft_` + // storage path (see RawAppDraft in sessions/appDraftCodec.ts). Must + // round-trip through chat writes or an edit erases the chosen name. + draft_path?: string } export type ResourceDraftState = { @@ -99,6 +109,10 @@ export type VariableDraftState = { export type WorkspaceItem = { type: WorkspaceItemType path: string + /** Friendly display path for a draft parked at a `…/draft_` storage + * path (the draft value's `draft_path`). Display-only — `path` is the key + * drafts are stored and routed under. */ + draftPath?: string summary?: string language?: ScriptLang triggerKind?: TriggerKind @@ -115,6 +129,8 @@ export type WorkspaceItem = { | CreateResource | CreateVariable | AppDraftValue + /** Input schema of a script read (flows carry theirs inside `value`). */ + schema?: unknown isDraft: boolean isLiveDraft?: boolean } diff --git a/frontend/src/lib/components/copilot/chat/imageUtils.test.ts b/frontend/src/lib/components/copilot/chat/imageUtils.test.ts new file mode 100644 index 0000000000..136c556258 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/imageUtils.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from 'vitest' +import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' +import { + boundImagePartBytes, + captureScale, + fileToAttachedImage, + imagesFromContent, + MAX_IMAGE_BYTES, + messagesHaveImageParts, + parseImageDataUrl, + stripImagePartsFromMessages +} from './imageUtils' + +describe('fileToAttachedImage size bound', () => { + // Decoding allocates ~4 bytes per pixel before the downscale can run, so an + // oversized file must be refused up front. + it('rejects a file over the byte cap', async () => { + const blob = { size: MAX_IMAGE_BYTES + 1, type: 'image/png' } as unknown as Blob + await expect(fileToAttachedImage(blob)).rejects.toThrow(/too large/i) + }) +}) + +describe('parseImageDataUrl', () => { + it('splits media type and base64 payload', () => { + expect(parseImageDataUrl('data:image/png;base64,AAAA')).toEqual({ + mediaType: 'image/png', + base64: 'AAAA' + }) + expect(parseImageDataUrl('data:image/jpeg;base64,ZZ==')).toEqual({ + mediaType: 'image/jpeg', + base64: 'ZZ==' + }) + }) + + it('defaults to png and empty payload on a malformed url', () => { + expect(parseImageDataUrl('not-a-data-url')).toEqual({ mediaType: 'image/png', base64: '' }) + }) +}) + +describe('stripImagePartsFromMessages', () => { + it('replaces image parts with a placeholder and collapses to a string', () => { + const messages: ChatCompletionMessageParam[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'look at this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,HUGEBLOB' } } + ] + } as any + ] + const out = stripImagePartsFromMessages(messages) + expect(out[0].content).toBe('look at this\n[image omitted]') + }) + + it('leaves image-free messages untouched (same reference)', () => { + const messages: ChatCompletionMessageParam[] = [{ role: 'user', content: 'plain' }] + const out = stripImagePartsFromMessages(messages) + expect(out[0]).toBe(messages[0]) + }) +}) + +describe('boundImagePartBytes', () => { + const imgMsg = (payloadChars: number, text: string): ChatCompletionMessageParam => + ({ + role: 'user', + content: [ + { type: 'text', text }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,' + 'A'.repeat(payloadChars) } + } + ] + }) as any + + it('returns the same array when everything fits', () => { + const messages = [imgMsg(100, 'a')] + expect(boundImagePartBytes(messages, 1000)).toBe(messages) + }) + + const imageParts = (m: ChatCompletionMessageParam) => + (m.content as any[]).filter((p) => p?.type === 'image_url').length + + it('strips the oldest images first once the cap is exceeded', () => { + // 1000 base64 chars ≈ 750 bytes each: the newest fits alone, both together don't + const messages = [ + imgMsg(1000, 'old'), + { role: 'assistant', content: 'ok' } as ChatCompletionMessageParam, + imgMsg(1000, 'new') + ] + const out = boundImagePartBytes(messages, 1000) + expect(imageParts(out[0])).toBe(0) + expect(JSON.stringify(out[0].content)).toContain('[image omitted]') + expect(out[1]).toBe(messages[1]) + expect(imageParts(out[2])).toBe(1) + }) + + // An over-cap batch on the CURRENT turn must keep the subset that fits, not + // silently send a text-only message while the composer showed attached images. + // The newest parts win: for screenshot follow-ups the last image is the app's + // current state. + it('keeps the newest fitting subset when the newest message alone exceeds the cap', () => { + const url = (marker: string) => ({ + type: 'image_url', + image_url: { url: 'data:image/png;base64,' + marker.repeat(1000) } + }) + const messages = [ + { + role: 'user', + content: [{ type: 'text', text: 'batch' }, url('A'), url('B'), url('C')] + } as any + ] + const out = boundImagePartBytes(messages, 1600) + // 750 bytes each against a 1600-byte cap: the two NEWEST fit, the oldest drops + const content = out[0].content as any[] + expect(content[1]).toEqual({ type: 'text', text: '[image omitted]' }) + expect(content[2].image_url.url).toContain('B') + expect(content[3].image_url.url).toContain('C') + }) +}) + +describe('messagesHaveImageParts', () => { + it('detects an image part anywhere in the history', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'plain' }, + { + role: 'user', + content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,A' } }] + } as any + ] + expect(messagesHaveImageParts(messages)).toBe(true) + }) + + it('is false for string content and image-free part arrays', () => { + const messages: ChatCompletionMessageParam[] = [ + { role: 'user', content: 'plain' }, + { role: 'user', content: [{ type: 'text', text: 'also plain' }] } as any + ] + expect(messagesHaveImageParts(messages)).toBe(false) + }) +}) + +describe('imagesFromContent', () => { + it('recovers image parts and skips text (including the omitted placeholder)', () => { + const content = [ + { type: 'text', text: 'look' }, + { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,AAAA' } }, + { type: 'text', text: '[image omitted]' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,BBBB' } } + ] + expect(imagesFromContent(content)).toEqual([ + { dataUrl: 'data:image/jpeg;base64,AAAA', mediaType: 'image/jpeg' }, + { dataUrl: 'data:image/png;base64,BBBB', mediaType: 'image/png' } + ]) + }) + + it('is undefined for string content and image-free part arrays', () => { + expect(imagesFromContent('plain')).toBeUndefined() + expect(imagesFromContent([{ type: 'text', text: 'plain' }])).toBeUndefined() + }) +}) + +describe('captureScale', () => { + it('captures small targets above CSS resolution, capped at 2x', () => { + expect(captureScale(400)).toBe(2) + }) + + it('never yields a raster larger than MAX_IMAGE_EDGE, even below 1x', () => { + // A tall scrolling app body: rasterising at >=1x would allocate an + // unbounded canvas only for normalize to shrink or reject it. + expect(captureScale(10_000) * 10_000).toBe(1568) + expect(captureScale(10_000)).toBeLessThan(1) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/imageUtils.ts b/frontend/src/lib/components/copilot/chat/imageUtils.ts new file mode 100644 index 0000000000..eb8dd3b8ec --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/imageUtils.ts @@ -0,0 +1,273 @@ +/** + * Image handling shared by the two multimodal chat features: user-attached images + * (drag/drop/paste, GLOBAL mode) and the app agent's `take_screenshot` tool. + * + * Every image the model sees passes through here first so it is bounded in BOTH + * dimensions (≤ MAX_EDGE longest side — beyond this the provider downscales anyway + * and just bills more tokens) and bytes. Bounding bytes matters because the data URL + * rides every request (stateless APIs resend the whole history) and is persisted in + * the chat history's blob store. Everything is rasterised to PNG/JPEG so exotic + * inputs (SVG, WebP, HEIC where the browser can decode it) become a media type all + * providers accept. + */ +import type { + ChatCompletionContentPartImage, + ChatCompletionMessageParam +} from 'openai/resources/index.mjs' + +/** Longest-edge cap. Matches the point past which vision models downscale server-side. */ +export const MAX_IMAGE_EDGE = 1568 +/** Above this many bytes a PNG re-encodes to JPEG to keep history/storage bounded. */ +const PNG_SIZE_CAP = 700_000 +/** + * Refuse a file this large before reading it. Decoding allocates ~4 bytes per pixel + * — a 12MP photo is ~48MB of bitmap — and the downscale below can only run once that + * bitmap exists, so the cap has to bite before the read, not after. + */ +export const MAX_IMAGE_BYTES = 20_000_000 +/** Decoded-pixel ceiling, in case a small file expands to an absurd bitmap. */ +const MAX_IMAGE_PIXELS = 40_000_000 +/** + * Images one message may carry. Enforced wherever a message is assembled, not just + * at the composer: queuing clears the composer, so its own count would reset and let + * repeated sends stack an unbounded batch into a single message. + */ +export const MAX_ATTACHED_IMAGES = 8 + +export type ImageMediaType = 'image/png' | 'image/jpeg' + +/** A model-ready image: a normalised (bounded, png/jpeg) data URL plus its media type. */ +export type AttachedImage = { + dataUrl: string + mediaType: ImageMediaType + /** Original filename when it came from a user file; absent for screenshots. */ + name?: string +} + +/** Stands in for a stripped or evicted image part in message content. */ +export const IMAGE_OMITTED_PLACEHOLDER = '[image omitted]' + +/** + * Recover the model's own images from an API message's content parts. Anything + * resending a turn (retry, edit) must read images from here, never from the + * transcript bubble: a provider rejection strips them from history while the + * bubble keeps its copy so the user can still see what they sent — resending + * that copy would re-attach the image the provider just refused. + */ +export function imagesFromContent(content: unknown): AttachedImage[] | undefined { + if (!Array.isArray(content)) return undefined + const images = (content as any[]).flatMap((part): AttachedImage[] => { + if (part?.type !== 'image_url' || typeof part?.image_url?.url !== 'string') return [] + const dataUrl = part.image_url.url as string + return [ + { + dataUrl, + mediaType: + parseImageDataUrl(dataUrl).mediaType === 'image/jpeg' ? 'image/jpeg' : 'image/png' + } + ] + }) + return images.length > 0 ? images : undefined +} + +/** + * Raster scale for a DOM screenshot of a target whose longest CSS edge is + * `cssEdge`. Above CSS resolution (up to 2×) for small targets — the SVG + * re-render is vector, so the extra scale is real detail, not interpolation — + * but never a raster larger than MAX_IMAGE_EDGE: normalize would downscale the + * excess away, and rasterising an oversized body (a tall scrolling app) at ≥1× + * first can allocate a tab-freezing canvas. Sub-1× output is deliberate. + */ +export function captureScale(cssEdge: number): number { + return Math.min(2, MAX_IMAGE_EDGE / Math.max(1, cssEdge)) +} + +export function isImageFile(file: File | Blob): boolean { + return typeof file.type === 'string' && file.type.startsWith('image/') +} + +/** Byte size of a base64 data URL's payload (4 base64 chars → 3 bytes). */ +function base64Bytes(dataUrl: string): number { + const comma = dataUrl.indexOf(',') + const b64 = comma >= 0 ? dataUrl.slice(comma + 1) : dataUrl + const padding = b64.endsWith('==') ? 2 : b64.endsWith('=') ? 1 : 0 + return Math.max(0, Math.floor((b64.length * 3) / 4) - padding) +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image() + img.onload = () => resolve(img) + img.onerror = () => reject(new Error('Could not decode image')) + img.src = src + }) +} + +function blobToDataUrl(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.onerror = () => reject(reader.error ?? new Error('Could not read file')) + reader.readAsDataURL(blob) + }) +} + +/** PNG by default (lossless — crisp for the common UI-screenshot/diagram case); fall + * back to JPEG only when the PNG would blow the size cap (photographic content). */ +function encodeCanvas(canvas: HTMLCanvasElement): { dataUrl: string; mediaType: ImageMediaType } { + const png = canvas.toDataURL('image/png') + if (base64Bytes(png) <= PNG_SIZE_CAP) { + return { dataUrl: png, mediaType: 'image/png' } + } + // JPEG has no alpha channel and canvas encoders composite transparent pixels + // onto black, which hides dark strokes in a transparent diagram. Flatten onto + // white before encoding. + const flat = document.createElement('canvas') + flat.width = canvas.width + flat.height = canvas.height + const ctx = flat.getContext('2d') + if (ctx) { + ctx.fillStyle = '#ffffff' + ctx.fillRect(0, 0, flat.width, flat.height) + ctx.drawImage(canvas, 0, 0) + } + return { + dataUrl: (ctx ? flat : canvas).toDataURL('image/jpeg', 0.82), + mediaType: 'image/jpeg' + } +} + +/** + * Downscale a data URL to ≤ MAX_IMAGE_EDGE on its longest side and re-encode to + * png/jpeg. Used by both the file-attach path and the screenshot tool. + */ +export async function normalizeImageDataUrl( + dataUrl: string, + name?: string, + maxEdge: number = MAX_IMAGE_EDGE +): Promise { + const img = await loadImage(dataUrl) + const srcW = img.naturalWidth || img.width + const srcH = img.naturalHeight || img.height + if (!srcW || !srcH) throw new Error('Image has no dimensions') + if (srcW * srcH > MAX_IMAGE_PIXELS) throw new Error('Image resolution is too large') + const scale = Math.min(1, maxEdge / Math.max(srcW, srcH)) + const w = Math.max(1, Math.round(srcW * scale)) + const h = Math.max(1, Math.round(srcH * scale)) + const canvas = document.createElement('canvas') + canvas.width = w + canvas.height = h + const ctx = canvas.getContext('2d') + if (!ctx) throw new Error('Canvas 2D context unavailable') + ctx.drawImage(img, 0, 0, w, h) + return { ...encodeCanvas(canvas), name } +} + +/** Read a user-provided image file and produce a bounded, model-ready AttachedImage. */ +export async function fileToAttachedImage(file: File | Blob): Promise { + if (file.size > MAX_IMAGE_BYTES) throw new Error('Image file is too large') + const name = file instanceof File ? file.name : undefined + const dataUrl = await blobToDataUrl(file) + return await normalizeImageDataUrl(dataUrl, name) +} + +/** Split a data URL into its media type and base64 payload (for the Anthropic converter). */ +export function parseImageDataUrl(url: string): { mediaType: string; base64: string } { + const match = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url) + if (!match) return { mediaType: 'image/png', base64: '' } + return { mediaType: match[1] || 'image/png', base64: match[2] ? match[3] : '' } +} + +/** Build the OpenAI-format image content part that all three provider paths convert from. */ +export function dataUrlToImagePart(dataUrl: string): ChatCompletionContentPartImage { + return { type: 'image_url', image_url: { url: dataUrl } } +} + +/** Whether any message still carries an image_url content part. */ +export function messagesHaveImageParts(messages: ChatCompletionMessageParam[]): boolean { + return messages.some( + (message) => + Array.isArray(message.content) && + (message.content as any[]).some((part) => part?.type === 'image_url') + ) +} + +/** + * Total decoded image bytes one request may carry. Providers reject the whole + * request body over a size limit (20MB on Bedrock, 32MB direct Anthropic), and + * that 413 never mentions images, so the vision-rejection fallback cannot + * recover it — each request must stay under the limit in the first place. + * Compaction cannot be relied on for this: it triggers on estimated tokens, + * and images are cheap in tokens relative to their bytes. 12MB decoded is + * ~16MB of base64 on the wire, safely under the tightest limit with text. + */ +export const MAX_TOTAL_IMAGE_BYTES = 12_000_000 + +/** + * Keep the request's cumulative image bytes under the cap by stripping the + * OLDEST image parts first (the newest images are the ones the conversation + * is about). Part-granular so a single over-cap batch keeps the subset that + * fits — the newest message never silently loses all its images (one bounded + * image alone cannot exceed the cap). Returns the input array unchanged when + * everything fits. + */ +export function boundImagePartBytes( + messages: ChatCompletionMessageParam[], + cap: number = MAX_TOTAL_IMAGE_BYTES +): ChatCompletionMessageParam[] { + let total = 0 + const drops = new Map>() + for (let i = messages.length - 1; i >= 0; i--) { + const content = messages[i].content + if (!Array.isArray(content)) continue + // Parts walk in reverse too: within a message they are in attachment order, + // and for screenshot follow-ups the last one is the app's current state. + for (let j = (content as any[]).length - 1; j >= 0; j--) { + const part = (content as any[])[j] + if (part?.type !== 'image_url' || typeof part?.image_url?.url !== 'string') continue + total += base64Bytes(part.image_url.url) + if (total > cap) { + if (!drops.has(i)) drops.set(i, new Set()) + drops.get(i)!.add(j) + } + } + } + if (drops.size === 0) return messages + return messages.map((message, i) => { + const drop = drops.get(i) + if (!drop) return message + return { + ...message, + content: (message.content as any[]).map((part, j) => + drop.has(j) ? { type: 'text', text: IMAGE_OMITTED_PLACEHOLDER } : part + ) + } as ChatCompletionMessageParam + }) +} + +/** + * Replace image_url content parts with a short text placeholder, collapsing the + * remaining parts back to a plain string. Used to keep base64 blobs out of the + * summarizer request during compaction (the summary text then stands in for them). + */ +export function stripImagePartsFromMessages( + messages: ChatCompletionMessageParam[] +): ChatCompletionMessageParam[] { + return messages.map((message) => { + if (!Array.isArray(message.content)) return message + let hadImage = false + const text = (message.content as any[]) + .map((part) => { + if (part?.type === 'text') return part.text ?? '' + if (part?.type === 'image_url') { + hadImage = true + return IMAGE_OMITTED_PLACEHOLDER + } + return '' + }) + .filter(Boolean) + .join('\n') + if (!hadImage) return message + return { ...message, content: text } as ChatCompletionMessageParam + }) +} diff --git a/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts new file mode 100644 index 0000000000..588fe4f4ff --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/messageDraft.svelte.ts @@ -0,0 +1,152 @@ +/** + * A message draft: the four lanes that ship together with one send — text, + * pastes, images, text files. Every place a draft accumulates or moves + * (composer attach, queue append, dequeue restore, failure restore) goes + * through this type, so the draft rules — file dedupe by source identity, + * courtesy rename, attachment slot caps, all-lanes-move-together — live here + * once instead of being re-implemented at each aggregation point. + * + * Deliberately NOT owned here: the conversation byte budget (needs + * manager-wide state — enforced at the composer until it moves into the + * store) and @context/DOM picks (ContextManager owns their lifecycle). + */ +import { MAX_ATTACHED_IMAGES, type AttachedImage } from './imageUtils' +import type { PasteAttachment } from './pasteTokens' +import { + admitWithinByteBudget, + foldIntoDraft, + MAX_ATTACHED_FILES, + type AttachedTextFile +} from './textFileUtils' + +/** A draft's four lanes as plain data — what moves between owners. */ +export interface DraftSnapshot { + text: string + pastes: PasteAttachment[] + images: AttachedImage[] + files: AttachedTextFile[] +} + +export class MessageDraft { + text = $state('') + pastes = $state([]) + images = $state([]) + files = $state([]) + + constructor(seed?: Partial) { + if (seed?.text) this.text = seed.text + if (seed?.pastes) this.pastes = [...seed.pastes] + if (seed?.images) this.images = [...seed.images] + if (seed?.files) this.files = [...seed.files] + } + + get isEmpty(): boolean { + return ( + this.text.trim() === '' && + this.pastes.length === 0 && + this.images.length === 0 && + this.files.length === 0 + ) + } + + /** Files joining a draft always fold (dedupe by source identity, courtesy + * rename) and respect the slot cap. `byteBudget`, when given, admits the + * folded entries by their decoded size — the fold must run first because + * dedupe changes what gets charged. Returns dropped counts so the caller can + * toast — the draft has no UI. */ + addFiles( + reads: { name: string; content: string; sourceName?: string }[], + byteBudget?: number + ): { droppedAtCap: number; droppedAtBudget: number } { + let folded = foldIntoDraft(this.files, reads) + let droppedAtBudget = 0 + if (byteBudget !== undefined) { + const res = admitWithinByteBudget(folded, byteBudget) + folded = res.admitted + droppedAtBudget = res.dropped + } + const merged = [...this.files, ...folded] + const droppedAtCap = Math.max(0, merged.length - MAX_ATTACHED_FILES) + this.files = merged.slice(0, MAX_ATTACHED_FILES) + return { droppedAtCap, droppedAtBudget } + } + + /** Images join up to the slot cap. Returns the dropped count (caller toasts). */ + addImages(images: AttachedImage[]): number { + const merged = [...this.images, ...images] + const dropped = Math.max(0, merged.length - MAX_ATTACHED_IMAGES) + this.images = merged.slice(0, MAX_ATTACHED_IMAGES) + return dropped + } + + /** + * Merge a restored draft on top of this one (queued-message delete, restore + * after a cancelled/errored turn): the restored draft was written FIRST, so + * its text lands above and its attachments ahead of the newer ones — at the + * caps it is the newest additions that drop, never the restored draft. + * Returns whether text merged onto a non-empty draft (the caller must then + * keep both drafts' context), plus dropped counts for toasts. + */ + prepend(restored: { text: string; images?: AttachedImage[]; files?: AttachedTextFile[] }): { + mergedIntoDraft: boolean + droppedImages: number + droppedFiles: number + } { + const mergedIntoDraft = !!restored.text && !!this.text.trim() + // An attachment-only restore has empty text; prepending would only add blank lines. + if (restored.text) { + this.text = this.text.trim() ? `${restored.text}\n\n${this.text}` : restored.text + } + let droppedImages = 0 + if (restored.images?.length) { + const merged = [...restored.images, ...this.images] + droppedImages = Math.max(0, merged.length - MAX_ATTACHED_IMAGES) + this.images = merged.slice(0, MAX_ATTACHED_IMAGES) + } + let droppedFiles = 0 + if (restored.files?.length) { + // The restored entries were already a normalized draft; the current + // (newer) files fold against them so dedupe/rename still apply. + const merged = [...restored.files, ...foldIntoDraft(restored.files, this.files)] + droppedFiles = Math.max(0, merged.length - MAX_ATTACHED_FILES) + this.files = merged.slice(0, MAX_ATTACHED_FILES) + } + return { mergedIntoDraft, droppedImages, droppedFiles } + } + + /** Replace the draft with a snapshot, but only when it is empty — an occupied + * draft keeps what the user is writing. Returns whether the restore was taken. */ + replaceIfEmpty(snapshot: Partial): boolean { + if (!this.isEmpty) return false + this.replace(snapshot) + return true + } + + /** Unconditionally replace all lanes (put a taken queue back, etc.). */ + replace(snapshot: Partial): void { + this.text = snapshot.text ?? '' + this.pastes = [...(snapshot.pastes ?? [])] + this.images = [...(snapshot.images ?? [])] + this.files = [...(snapshot.files ?? [])] + } + + /** Snapshot and clear atomically — the four lanes always move together, so no + * call site can take one and forget another. */ + take(): DraftSnapshot { + const snapshot: DraftSnapshot = { + text: this.text, + pastes: this.pastes, + images: this.images, + files: this.files + } + this.clear() + return snapshot + } + + clear(): void { + this.text = '' + this.pastes = [] + this.images = [] + this.files = [] + } +} diff --git a/frontend/src/lib/components/copilot/chat/messageDraft.test.ts b/frontend/src/lib/components/copilot/chat/messageDraft.test.ts new file mode 100644 index 0000000000..a6a9bb6169 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/messageDraft.test.ts @@ -0,0 +1,91 @@ +import { describe, it, expect } from 'vitest' +import { MessageDraft } from './messageDraft.svelte' + +// Fold/rename/dedupe semantics are pinned in textFileUtils.test.ts; these pin +// the draft-level guarantees: lanes move together, restores respect occupancy, +// aggregation always applies the rules. + +describe('MessageDraft', () => { + it('take() snapshots and clears all four lanes atomically', () => { + const d = new MessageDraft({ + text: 'hello', + pastes: [{ id: 'p1', content: 'x' } as any], + images: [{ dataUrl: 'i1' } as any], + files: [{ name: 'a.md', content: 'a' }] + }) + const snap = d.take() + expect(snap.text).toBe('hello') + expect(snap.pastes).toHaveLength(1) + expect(snap.images).toHaveLength(1) + expect(snap.files).toHaveLength(1) + expect(d.isEmpty).toBe(true) + }) + + it('replaceIfEmpty declines when any lane is occupied', () => { + const d = new MessageDraft({ files: [{ name: 'a.md', content: 'a' }] }) + expect(d.replaceIfEmpty({ text: 'restored' })).toBe(false) + expect(d.files).toHaveLength(1) + d.clear() + expect(d.replaceIfEmpty({ text: 'restored' })).toBe(true) + expect(d.text).toBe('restored') + }) + + it('prepend puts the restored draft first and folds the newer files against it', () => { + const d = new MessageDraft({ text: 'typing', files: [{ name: 'a.md', content: 'a' }] }) + const res = d.prepend({ + text: 'restored', + files: [ + { name: 'a.md', content: 'a' }, // identical to the newer draft's copy → it dedupes + { name: 'a (2).md', content: 'b', sourceName: 'a.md' } + ] + }) + expect(res.mergedIntoDraft).toBe(true) + expect(d.text).toBe('restored\n\ntyping') + expect(d.files.map((f) => f.name)).toEqual(['a.md', 'a (2).md']) + }) + + it('prepend gives the restored draft chronological priority at the caps', () => { + // The restored draft was written first — the cap must drop the NEWEST + // additions, never the restored attachments. + const d = new MessageDraft({ + files: Array.from({ length: 6 }, (_, i) => ({ name: `new${i}.md`, content: `${i}` })) + }) + const res = d.prepend({ + text: '', + files: Array.from({ length: 3 }, (_, i) => ({ name: `old${i}.md`, content: `o${i}` })) + }) + expect(res.droppedFiles).toBe(1) + expect(d.files.map((f) => f.name)).toEqual([ + 'old0.md', + 'old1.md', + 'old2.md', + 'new0.md', + 'new1.md', + 'new2.md', + 'new3.md', + 'new4.md' + ]) + }) + + it('addFiles reports drops at the slot cap and the byte budget', () => { + const d = new MessageDraft() + const many = Array.from({ length: 10 }, (_, i) => ({ name: `${i}.md`, content: `${i}` })) + expect(d.addFiles(many).droppedAtCap).toBe(2) + expect(d.files).toHaveLength(8) + + const e = new MessageDraft() + // Budget admits by decoded size AFTER the fold — the identical duplicate is + // deduped, not charged. + const res = e.addFiles( + [ + { name: 'a.md', content: 'aaaa' }, + { name: 'a.md', content: 'aaaa' }, + { name: 'b.md', content: 'bbbb' }, + { name: 'c.md', content: 'cccc' } + ], + 8 + ) + expect(e.files.map((f) => f.name)).toEqual(['a.md', 'b.md']) + expect(res.droppedAtBudget).toBe(1) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.test.ts b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts new file mode 100644 index 0000000000..d93e383db1 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/openai-responses.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest' +import { openAIWebSearchDetails, toResponsesContent } from './openai-responses' + +// openai-responses.ts pulls in the chat client/registry layer at import time; the +// helper under test is pure, so stub those side-effecting modules away. +vi.mock('../lib', () => ({ + createOpenAIProxyClient: vi.fn(), + getAiProxyBaseURL: vi.fn(), + getProviderAndCompletionConfig: vi.fn(), + providerSupportsWebSearch: vi.fn(), + workspaceAIClients: {} +})) + +vi.mock('../reasoningRegistry', () => ({ + applyReasoningToConfig: vi.fn() +})) + +vi.mock('./shared', () => ({ + processToolCall: vi.fn(), + appendPendingToolImages: vi.fn() +})) + +describe('toResponsesContent', () => { + it('passes a plain string through unchanged', () => { + expect(toResponsesContent('hello')).toBe('hello') + }) + + it('maps text parts to input_text and image_url parts to input_image (string url)', () => { + const out = toResponsesContent([ + { type: 'text', text: 'describe this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,ZZZZ' } } + ]) as any[] + + expect(out).toEqual([ + { type: 'input_text', text: 'describe this' }, + { type: 'input_image', image_url: 'data:image/png;base64,ZZZZ' } + ]) + }) +}) + +describe('openAIWebSearchDetails', () => { + it('prefers the queries array over the deprecated singular query', () => { + expect( + openAIWebSearchDetails({ action: { type: 'search', queries: ['a', 'b'], query: 'old' } }) + ).toEqual({ query: 'a, b', sources: undefined }) + }) + + it('falls back to the singular query when queries is absent', () => { + expect(openAIWebSearchDetails({ action: { type: 'search', query: 'solo' } }).query).toBe('solo') + }) + + it('keeps only url-shaped sources and returns undefined query when neither field is usable', () => { + expect( + openAIWebSearchDetails({ + action: { type: 'search', queries: [], sources: [{ url: 'https://a.dev' }, { nope: 1 }] } + }) + ).toEqual({ query: undefined, sources: [{ url: 'https://a.dev' }] }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 426e1e76e2..9db5baf42f 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -13,7 +13,13 @@ import { workspaceAIClients } from '../lib' import { applyReasoningToConfig } from '../reasoningRegistry' -import { processToolCall, type Tool, type ToolCallbacks } from './shared' +import { + appendPendingToolImages, + processToolCall, + type Tool, + type ToolCallbacks, + type WebSearchSource +} from './shared' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -30,24 +36,80 @@ const openAIWebSearchToolId = (itemId: string) => `openai_web_search:${itemId}` function setOpenAIWebSearchStatus( callbacks: ToolCallbacks & { onMessageEnd: () => void }, itemId: string, - status: WebSearchStatus + status: WebSearchStatus, + details?: { query?: string; sources?: WebSearchSource[] } ) { const isLoading = status === 'in_progress' || status === 'searching' const failed = status === 'failed' + const sources = details?.sources callbacks.onMessageEnd() callbacks.setToolStatus(openAIWebSearchToolId(itemId), { - content: failed ? 'Web search failed' : isLoading ? 'Searching the web...' : 'Searched the web', + content: failed + ? 'Web search failed' + : isLoading + ? 'Searching the web...' + : details?.query + ? `Searched the web for "${details.query}"` + : 'Searched the web', error: failed ? 'Web search failed' : undefined, isLoading, isStreamingArguments: false, needsConfirmation: false, toolName: 'web_search', - showDetails: false, - autoCollapseDetails: true + // Sources keep the card expanded (no auto-collapse) so the consulted + // pages surface live as each search completes mid-stream. + ...(sources?.length + ? { webSearchSources: sources, showDetails: true, autoCollapseDetails: false } + : {}) }) } +// Pull query + consulted URLs out of a completed web_search_call item. +// The pinned SDK types the action shapes (ResponseFunctionWebSearch.Search) +// but its ResponseFunctionWebSearch interface predates the `action` property +// itself, so the field must be read untyped and shape-checked. +export function openAIWebSearchDetails(item: any): { + query?: string + sources?: WebSearchSource[] +} { + const action = item?.action + // The current schema sends a `queries` array and may omit the deprecated + // singular `query`; support both so the label never falls back to the bare + // "Searched the web". + const queries: string[] = Array.isArray(action?.queries) + ? action.queries.filter((q: any) => typeof q === 'string' && q) + : [] + const query = queries.length + ? queries.join(', ') + : typeof action?.query === 'string' && action.query + ? action.query + : undefined + const sources = Array.isArray(action?.sources) + ? action.sources + .filter((s: any) => typeof s?.url === 'string') + .map((s: any) => ({ url: s.url })) + : undefined + return { query, sources } +} + // Conversion utilities for Responses API + +/** + * Translate Chat-Completions message content to Responses-native content. Strings + * pass through; a content-part array maps text→input_text and image_url→input_image + * (Responses takes image_url as a plain string, not the {url} object). + */ +export function toResponsesContent(content: unknown): unknown { + if (!Array.isArray(content)) return content + return content.map((part) => { + if (part?.type === 'text') return { type: 'input_text', text: part.text } + if (part?.type === 'image_url' && part.image_url?.url) { + return { type: 'input_image', image_url: part.image_url.url } + } + return part + }) +} + function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]): { instructions?: string input: Array @@ -100,7 +162,7 @@ function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]) input.push({ type: 'message' as const, role: m.role === 'developer' ? 'developer' : m.role === 'assistant' ? 'assistant' : 'user', - content: m.content + content: toResponsesContent(m.content) }) } } @@ -159,6 +221,7 @@ export async function getOpenAIResponsesCompletion( openaiClient?: OpenAI webSearch?: boolean reasoningEffort?: string + reasoningSummary?: boolean } ) { const { provider, config } = getProviderAndCompletionConfig({ @@ -174,10 +237,20 @@ export async function getOpenAIResponsesCompletion( options?.reasoningEffort ) + // Reasoning summaries make the model's thinking renderable in the chat, but + // OpenAI rejects the request (400 on reasoning.summary) for organizations + // that haven't completed verification — callers opt in and fall back. + if (options?.reasoningSummary && responsesConfig.reasoning) { + responsesConfig.reasoning = { ...responsesConfig.reasoning, summary: 'auto' } + } + // Enable OpenAI's built-in web search tool. The proxy forwards the body - // verbatim, so this reaches OpenAI as a native server-side tool. + // verbatim, so this reaches OpenAI as a native server-side tool. Sources + // (the URLs each search consulted) are only returned when asked for via + // `include` — they feed the expandable source list on the tool card. if (options?.webSearch && providerSupportsWebSearch(provider)) { responsesConfig.tools = [...(responsesConfig.tools ?? []), { type: 'web_search' }] + responsesConfig.include = [...(responsesConfig.include ?? []), 'web_search_call.action.sources'] } const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() @@ -291,6 +364,20 @@ export async function parseOpenAIResponsesCompletion( textContent += event.delta }) + // Stream the reasoning summary (present when the request asked for + // reasoning.summary) into the thinking display. Summaries arrive as + // separate parts; join them as paragraphs. + let reasoningSummaryParts = 0 + runner.on('response.reasoning_summary_part.added', () => { + reasoningSummaryParts++ + if (reasoningSummaryParts > 1) { + callbacks.onReasoningDelta?.('\n\n') + } + }) + runner.on('response.reasoning_summary_text.delta', (event) => { + callbacks.onReasoningDelta?.(event.delta) + }) + // Handle new output items (including function calls) runner.on('response.output_item.added', (event) => { const item = event.item @@ -315,7 +402,7 @@ export async function parseOpenAIResponsesCompletion( callbacks.onMessageEnd() callbacks.setToolStatus(`${item.id}`, { isLoading: true, - content: `Calling ${item.name}...`, + content: tool?.streamingLabel ?? `Calling ${item.name}...`, toolName: item.name, isStreamingArguments: shouldStream, showFade: tool?.showFade, @@ -339,6 +426,25 @@ export async function parseOpenAIResponsesCompletion( setOpenAIWebSearchStatus(callbacks, event.item_id, 'completed') }) + // The completed event above only carries item_id; the full item (with + // action.query and the requested action.sources) lands in output_item.done, + // mid-stream — surface the source list there rather than at end of turn. + // Track surfaced ids so the final-response sweep doesn't re-emit the status + // and re-expand a card the user collapsed in the meantime. + const surfacedWebSearchCalls = new Set() + runner.on('response.output_item.done', (event) => { + const item = event.item as any + if (item?.type === 'web_search_call' && item.id) { + surfacedWebSearchCalls.add(item.id) + setOpenAIWebSearchStatus( + callbacks, + item.id, + item.status ?? 'completed', + openAIWebSearchDetails(item) + ) + } + }) + // Stream function call arguments incrementally runner.on('response.function_call_arguments.delta', (event) => { if (currentStreamingTool?.shouldStream && currentStreamingTool.itemId === event.item_id) { @@ -414,8 +520,9 @@ export async function parseOpenAIResponsesCompletion( const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage) for (const item of finalResponse.output ?? []) { - if (item.type === 'web_search_call') { - setOpenAIWebSearchStatus(callbacks, item.id, item.status) + if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) { + // Fallback for a call whose output_item.done event was missed. + setOpenAIWebSearchStatus(callbacks, item.id, item.status, openAIWebSearchDetails(item)) } } @@ -440,6 +547,7 @@ export async function parseOpenAIResponsesCompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) return { shouldContinue: true, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts index 4f3e31986d..c0d9b7ac8b 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.test.ts @@ -57,10 +57,11 @@ function makeHelpers(overrides: Partial = {}): { }, proposeNode: async (input) => { calls.proposeNode = [...(calls.proposeNode ?? []), [input]] - return { path: input.path } + return { path: input.path, detectedReads: [], detectedWrites: [] } }, editNode: async (path, content) => { calls.editNode = [...(calls.editNode ?? []), [path, content]] + return { detectedReads: [], detectedWrites: [] } }, removeProposedNode: record('removeProposedNode'), testNode: async () => 'job-123', @@ -115,6 +116,40 @@ describe('pipeline tools', () => { expect(out).toContain('not deployed') }) + it('build_pipeline_node reports the inferred asset lineage', async () => { + const { helpers } = makeHelpers({ + proposeNode: async (input) => ({ + path: input.path, + detectedReads: ['s3:///raw/in.csv'], + detectedWrites: ['ducklake://main/out'] + }) + }) + const out = await toolByName('build_pipeline_node').fn({ + args: { path: 'f/analytics/clean', language: 'duckdb', content: '-- pipeline' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(out).toContain('writes ducklake://main/out') + expect(out).toContain('reads s3:///raw/in.csv') + }) + + it('build_pipeline_node warns when no asset lineage is inferred', async () => { + const { helpers } = makeHelpers({ + proposeNode: async (input) => ({ path: input.path, detectedReads: [], detectedWrites: [] }) + }) + const out = await toolByName('build_pipeline_node').fn({ + args: { path: 'f/analytics/clean', language: 'duckdb', content: '-- pipeline' }, + workspace: 'w', + helpers, + toolCallbacks: noopCallbacks(), + toolId: 't' + }) + expect(out).toMatch(/no storage-asset read or write was inferred/i) + expect(out).toContain('string literal') + }) + it('edit_pipeline_node reads then applies an exact find/replace', async () => { const { helpers, calls } = makeHelpers({ getNodeBody: async () => ({ language: 'bun', content: 'const x = 1\nconst y = 2\n' }) diff --git a/frontend/src/lib/components/copilot/chat/pipeline/core.ts b/frontend/src/lib/components/copilot/chat/pipeline/core.ts index 86fe9eb634..f689699dd5 100644 --- a/frontend/src/lib/components/copilot/chat/pipeline/core.ts +++ b/frontend/src/lib/components/copilot/chat/pipeline/core.ts @@ -53,15 +53,21 @@ export interface PipelineAIChatHelpers { getPipelineContext: () => PipelineContext /** Read a node's source (the in-flight draft body if one exists, else deployed). */ getNodeBody: (path: string) => Promise<{ language: ScriptLang; content: string } | undefined> - /** Create a brand-new pipeline node as an unsaved draft on the canvas. */ + /** Create a brand-new pipeline node as an unsaved draft on the canvas. + * Returns the asset lineage inferred from the node (as URIs) so the caller can + * confirm the intended edges formed without a separate graph read. */ proposeNode: (input: { path: string language: ScriptLang content: string outputKind?: PipelineOutputKind - }) => Promise<{ path: string }> - /** Replace an existing node's body, applied as an unsaved draft. */ - editNode: (path: string, content: string) => Promise + }) => Promise<{ path: string; detectedReads: string[]; detectedWrites: string[] }> + /** Replace an existing node's body, applied as an unsaved draft. Returns the + * re-inferred asset lineage (URIs) so the caller sees the effect of the edit. */ + editNode: ( + path: string, + content: string + ) => Promise<{ detectedReads: string[]; detectedWrites: string[] }> /** Discard the unsaved draft at a path (undo a build_pipeline_node). */ removeProposedNode: (path: string) => Promise /** Preview-run a node (draft body preferred). Returns the started job id. */ @@ -184,9 +190,20 @@ const testPipelineNodeToolDef = createToolDef( { strict: false } ) -// ---------------------------------------------------------------------------- -// Tool set -// ---------------------------------------------------------------------------- +// Summarize the asset lineage the parser inferred from a just-applied node so the +// model gets same-turn feedback on whether its intended edges formed. An empty +// result is the useful signal: a write/read expressed via a variable or dynamic +// path is not detected, so no lineage edge forms — flag it rather than let the +// model discover it only on a later graph read. +function inferredLineageNote(reads: string[], writes: string[]): string { + const parts: string[] = [] + if (writes.length) parts.push(`writes ${writes.join(', ')}`) + if (reads.length) parts.push(`reads ${reads.join(', ')}`) + if (parts.length === 0) { + return ' No storage-asset read or write was inferred from it — if this node is meant to feed or consume another node, make sure the asset reference is a string literal (a variable, f-string, or computed path is not detected, so no lineage edge forms).' + } + return ` Inferred lineage: ${parts.join('; ')}.` +} export const pipelineTools: Tool[] = [ { @@ -225,7 +242,7 @@ export const pipelineTools: Tool[] = [ const pipeline = requirePipeline(helpers) const { path, language, content, output_kind } = buildPipelineNodeSchema.parse(args) toolCallbacks.setToolStatus(toolId, { content: `Building node '${path}'...` }) - await pipeline.proposeNode({ + const { detectedReads, detectedWrites } = await pipeline.proposeNode({ path, language: language as ScriptLang, content, @@ -235,7 +252,7 @@ export const pipelineTools: Tool[] = [ content: `Added draft node '${path}'`, result: 'Success' }) - return `Pipeline node '${path}' added as an unsaved draft on the canvas. It is not deployed — the user deploys it.` + return `Pipeline node '${path}' added as an unsaved draft on the canvas. It is not deployed — the user deploys it.${inferredLineageNote(detectedReads, detectedWrites)}` } }, { @@ -258,12 +275,12 @@ export const pipelineTools: Tool[] = [ replace_all ?? false, 'node source' ) - await pipeline.editNode(path, updated) + const { detectedReads, detectedWrites } = await pipeline.editNode(path, updated) toolCallbacks.setToolStatus(toolId, { content: `Edited draft '${path}'`, result: 'Success' }) - return `Pipeline node '${path}' updated as an unsaved draft on the canvas (not deployed).` + return `Pipeline node '${path}' updated as an unsaved draft on the canvas (not deployed).${inferredLineageNote(detectedReads, detectedWrites)}` } }, { @@ -319,8 +336,9 @@ export function getPipelinePromptSection(ctx: PipelineContext): string { Data Pipeline editor (ACTIVE): - The user has the /pipeline/${ctx.folder} editor open. A pipeline is a DAG of scripts (nodes) connected by storage assets (DuckLake tables, data tables, S3 objects, volumes, resources) and execution triggers. - Annotations are top-of-file comments in the NODE'S OWN comment syntax: \`--\` for SQL (duckdb/postgresql), \`#\` for python3/bash, \`//\` for bun/TS. The \`//\` shown below is the TS form — translate it (a \`// pipeline\` line in a SQL node is a syntax error that won't deploy). -- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`. +- A script becomes a pipeline node when its source starts with the \`// pipeline\` annotation. Declare execution-DAG inputs with \`// on \` (e.g. \`// on ducklake://main/orders\`). Outputs are inferred from what the body writes (wmill SDK calls / SQL CREATE TABLE / writeS3File); declare a managed output with \`// materialize \`. Optional badges: \`// partitioned \`, \`// freshness \`, \`// tag \`, \`// retry [delay]\`, \`// data_test ...\`, \`// measure = [where ]\`, \`// dimension = \`. - \`materialize\` (the managed output): \`// materialize \` means the runtime writes the node's output table FOR you — write the body as a single SELECT and the runtime wraps it in the create/replace, so do NOT also write your own CREATE TABLE / INSERT. IMPORTANT: \`// materialize\` is **DuckDB-only** and its target MUST be a DuckLake table (\`ducklake:///
\`) — deploy rejects it on any other language or target. For a \`python3\`/\`bun\`/\`postgresql\` node, do NOT use \`// materialize\`; write the output via the SDK instead (e.g. \`wmill.writeS3File(...)\`, a \`CREATE TABLE\` in postgresql, or \`wmill.databaseUrlFromResource\`/ducklake helpers) and let the output be inferred. Reach for \`duckdb\` when a node should materialize a DuckLake table. Write strategy: with no option it REPLACES the whole table each run (full refresh; the only mode whose output columns may change); \`// materialize append\` INSERT-appends rows (incremental); \`// materialize key=\` merges/upserts on \`\`. \`// materialize manual \` opts OUT of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. \`materialize\` is paired with partitioning for incremental pipelines: a \`// partitioned \` node runs once per partition (append/merge into a fixed-schema table), and the \`{partition}\` token — usable in any asset URI AND in the body SQL — is substituted with the current partition's IDENTITY string at run time. To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp with the exact identity format the runtime used (daily/hourly/weekly/monthly), so it always matches and you never hand-write a \`strftime\` format. Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at runtime. For \`dynamic\` partitioning the identity is your caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output DECLARATION on the node — it is not a command; there is no "materialize run". +- \`measure\` / \`dimension\` (declared metrics): on a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents can reuse the definition instead of re-deriving it and silently disagreeing. Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates sit under one GROUP BY. DuckLake-only, and only meaningful next to \`// materialize\`. Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures, an obvious \`count(*)\` earns nothing. To USE a metric another node declares, read that node with read_pipeline_node and reuse its exact expression rather than guessing it. - Use get_pipeline_graph to see the current nodes/assets/triggers, and read_pipeline_node before editing one. - Build new nodes with build_pipeline_node and edit existing ones with edit_pipeline_node. These apply directly as unsaved drafts on the canvas (like the flow/script editor applies AI edits) — they DO NOT deploy. There is no separate Accept/Reject step. Prefer these over the generic write_script/edit_script draft tools while a pipeline is open. - Reuse existing asset paths from the graph when wiring a downstream node to an upstream one (read the upstream's write asset, then \`// on\` that same URI). diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index e706fe11f5..f43b5ed370 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -1,7 +1,6 @@ import { ResourceService, JobService } from '$lib/gen/services.gen' import type { AIProvider, AIProviderModel, ResourceType, ScriptLang } from '$lib/gen/types.gen' import { capitalize, isObject, toCamel } from '$lib/utils' -import { get } from 'svelte/store' import { compile, phpCompile, pythonCompile } from '../../utils' import type { ChatCompletionSystemMessageParam, @@ -460,10 +459,18 @@ export const resourceTypeTool: Tool = { } } -// Generic DB schema tool factory that can be used by both script and flow modes -export function createDbSchemaTool(): Tool { +// Generic DB schema tool factory shared by the script, flow and global modes +export function createDbSchemaTool( + opts: { description?: string; updateEditorCache?: boolean } = {} +): Tool { + const { description, updateEditorCache = true } = opts return { - def: DB_SCHEMA_FUNCTION_DEF, + def: description + ? { + ...DB_SCHEMA_FUNCTION_DEF, + function: { ...DB_SCHEMA_FUNCTION_DEF.function, description } + } + : DB_SCHEMA_FUNCTION_DEF, fn: async ({ args, workspace, toolCallbacks, toolId }) => { if (!args.resourcePath) { throw new Error('Database path not provided') @@ -475,23 +482,24 @@ export function createDbSchemaTool(): Tool { workspace: workspace, path: args.resourcePath }) - const newDbSchemas = { - [args.resourcePath]: await getDbSchemas( - resource.resource_type, - args.resourcePath, - workspace, - (error) => { - console.error(error) - } - ) - } - dbSchemas.update((schemas) => ({ ...schemas, ...newDbSchemas })) - const dbs = get(dbSchemas) - const db = dbs[args.resourcePath] - if (!db) { + const dbSchema = await getDbSchemas( + resource.resource_type, + args.resourcePath, + workspace, + (error) => { + console.error(error) + } + ) + if (!dbSchema) { throw new Error('Database not found') } - const stringSchema = await formatDBSchema(db) + // The dbSchemas store is an editor cache keyed by resource path with no + // workspace dimension: a chat that may operate on a different workspace than + // the navigation one (global/session) must not write into it. + if (updateEditorCache) { + dbSchemas.update((schemas) => ({ ...schemas, [args.resourcePath]: dbSchema })) + } + const stringSchema = await formatDBSchema(dbSchema) toolCallbacks.setToolStatus(toolId, { content: 'Retrieved database schema for ' + args.resourcePath }) @@ -529,7 +537,9 @@ export async function searchExternalIntegrationResources(args: { query: string } return JSON.stringify(packagesSearchCache.get(args.query)) } - const result = await fetch(`https://registry.npmjs.org/-/v1/search?text=${args.query}&size=2`) + const result = await fetch( + `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(args.query)}&size=2` + ) const data = await result.json() const filtered = data.objects.filter( (r: PackageSearchQuery) => r.searchScore >= SCORE_THRESHOLD @@ -591,7 +601,8 @@ const SEARCH_NPM_PACKAGES_TOOL: ChatCompletionFunctionTool = { } } -export const searchNpmPackagesTool: Tool = { +// Helpers-agnostic so both script mode and global mode can offer it. +export const searchNpmPackagesTool: Tool<{}> = { def: SEARCH_NPM_PACKAGES_TOOL, fn: async ({ args, toolId, toolCallbacks }) => { toolCallbacks.setToolStatus(toolId, { content: 'Searching for relevant packages...' }) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 1b77e060c4..473c3d3335 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1,13 +1,29 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' import type { DisplayMessage, ToolDisplayMessage } from './shared' +import { openItemPreviewAction } from './shared' vi.mock('monaco-editor', () => ({ editor: {} })) +const userHolder = vi.hoisted(() => ({ + current: { is_super_admin: true } as { is_super_admin: boolean } +})) + vi.mock('$lib/stores', () => ({ - workspaceStore: { subscribe: () => () => undefined } + workspaceStore: { subscribe: () => () => undefined }, + userStore: { + subscribe: (run: (value: { is_super_admin: boolean }) => void) => { + run(userHolder.current) + return () => {} + } + } +})) + +vi.mock('$lib/components/triggers/email/utils', () => ({ + getEmailAddress: (localPart: string, _wlp: boolean, _wsId: string, domain: string) => + `${localPart}@${domain}` })) vi.mock('$lib/components/flows/flowTree', () => ({ @@ -30,7 +46,10 @@ vi.mock('$lib/gen', () => ({ MqttTriggerService: { createMqttTrigger: vi.fn() }, SqsTriggerService: { createSqsTrigger: vi.fn() }, GcpTriggerService: { createGcpTrigger: vi.fn() }, - AzureTriggerService: { createAzureTrigger: vi.fn() } + AzureTriggerService: { createAzureTrigger: vi.fn() }, + AmqpTriggerService: { createAmqpTrigger: vi.fn() }, + EmailTriggerService: { createEmailTrigger: vi.fn() }, + SettingService: { getGlobal: vi.fn() } })) vi.mock('$lib/utils', () => ({ @@ -211,6 +230,45 @@ describe('processToolCall', () => { expect(result.content).toBe(error) }) + it('surfaces the real error in the tool status when the tool throws', async () => { + const { createToolDef, processToolCall } = await import('./shared') + const apiError = Object.assign(new Error('Bad Request'), { + status: 400, + body: { error: { message: 'script not found at path f/scripts/missing' } } + }) + const setToolStatus = vi.fn() + + const result = await processToolCall({ + tools: [ + { + def: createToolDef(z.object({}), 'run_script', 'Run script'), + fn: vi.fn().mockRejectedValue(apiError) + } + ], + toolCall: { + id: 'call_err', + type: 'function', + function: { name: 'run_script', arguments: '{}' } + }, + helpers: {}, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus, + removeToolStatus: vi.fn() + } + }) + + const expectedError = 'script not found at path f/scripts/missing' + expect(setToolStatus).toHaveBeenLastCalledWith( + 'call_err', + expect.objectContaining({ + isLoading: false, + error: expectedError + }) + ) + expect(result.content).toBe(`Error while calling tool: ${expectedError}`) + }) + it('continues to confirmation when pre-confirmation validation passes', async () => { const { createToolDef, processToolCall } = await import('./shared') const fn = vi.fn().mockResolvedValue('ok') @@ -515,6 +573,118 @@ describe('processToolCall', () => { ) }) + it('email trigger: guides the user to set up email triggering when unconfigured', async () => { + const gen = (await import('$lib/gen')) as any + const { processToolCall } = await import('./shared') + const { createWorkspaceMutationTools } = await import('./workspaceTools') + const tools = createWorkspaceMutationTools() + + gen.SettingService.getGlobal.mockReset() + gen.SettingService.getGlobal.mockResolvedValue(null) + gen.EmailTriggerService.createEmailTrigger.mockReset() + + const call = (id: string) => + processToolCall({ + tools, + toolCall: { + id, + type: 'function', + function: { + name: 'create_trigger', + arguments: JSON.stringify({ + kind: 'email', + path: 'f/triggers/email_current', + config: { local_part: 'orders' } + }) + } + }, + helpers: { + getWorkspaceMutationTarget: () => ({ + kind: 'flow', + path: 'f/flows/current', + deployed: true + }) + }, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: vi.fn().mockResolvedValue(true) + } + }) + + userHolder.current = { is_super_admin: true } + const superadminResult = await call('call_email_super') + expect(gen.EmailTriggerService.createEmailTrigger).not.toHaveBeenCalled() + expect(superadminResult.content).toContain('not set up') + expect(superadminResult.content).toContain('As a superadmin') + + userHolder.current = { is_super_admin: false } + const memberResult = await call('call_email_member') + expect(gen.EmailTriggerService.createEmailTrigger).not.toHaveBeenCalled() + expect(memberResult.content).toContain('Ask an instance superadmin') + }) + + it('email trigger: creates it and reports the address when email triggering is configured', async () => { + const gen = (await import('$lib/gen')) as any + const { processToolCall } = await import('./shared') + const { createWorkspaceMutationTools } = await import('./workspaceTools') + const tools = createWorkspaceMutationTools() + + gen.SettingService.getGlobal.mockReset() + gen.SettingService.getGlobal.mockResolvedValue('mail.example.com') + gen.EmailTriggerService.createEmailTrigger.mockReset() + gen.EmailTriggerService.createEmailTrigger.mockResolvedValue('email-created') + + const result = await processToolCall({ + tools, + toolCall: { + id: 'call_email_ok', + type: 'function', + function: { + name: 'create_trigger', + arguments: JSON.stringify({ + kind: 'email', + path: 'f/triggers/email_current', + config: { local_part: 'orders' } + }) + } + }, + helpers: { + getWorkspaceMutationTarget: () => ({ + kind: 'flow', + path: 'f/flows/current', + deployed: true + }) + }, + workspace: 'test-workspace', + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + requestConfirmation: vi.fn().mockResolvedValue(true) + } + }) + + expect(gen.EmailTriggerService.createEmailTrigger).toHaveBeenCalledWith({ + workspace: 'test-workspace', + requestBody: expect.objectContaining({ + local_part: 'orders', + // defaulted before the request is sent; the backend column is NOT NULL + workspaced_local_part: false, + script_path: 'f/flows/current', + is_flow: true + }) + }) + expect(JSON.parse(result.content as string)).toEqual( + expect.objectContaining({ + success: true, + kind: 'email', + email_address: 'orders@mail.example.com', + backend_result: 'email-created' + }) + ) + }) + it('surfaces workspace mutation tool execution errors to the user', async () => { const gen = (await import('$lib/gen')) as any const { processToolCall } = await import('./shared') @@ -898,3 +1068,59 @@ describe('trimJob', () => { expect(job.result).toBe(42) }) }) + +describe('appendPendingToolImages', () => { + // Tool results are string-only, so tool-produced images ride a follow-up + // user message appended after the whole tool batch. It must land in BOTH + // arrays (messages = sent next iteration, addedMessages = committed to + // history) and drain the buffer exactly once — a second flush appending the + // same screenshots again would duplicate them in history. + it('appends one user message to both arrays and drains the buffer once', async () => { + const { appendPendingToolImages } = await import('./shared') + let pending = [{ dataUrl: 'data:image/png;base64,SHOT', mediaType: 'image/png' as const }] + const toolCallbacks = { + setToolStatus: vi.fn(), + takePendingToolImages: () => { + const taken = pending + pending = [] + return taken + } + } + const messages: any[] = [] + const addedMessages: any[] = [] + + appendPendingToolImages(messages, addedMessages, toolCallbacks as any) + + expect(messages).toHaveLength(1) + expect(messages[0]).toBe(addedMessages[0]) + expect(messages[0].role).toBe('user') + expect(messages[0].content[1]).toEqual({ + type: 'image_url', + image_url: { url: 'data:image/png;base64,SHOT' } + }) + + appendPendingToolImages(messages, addedMessages, toolCallbacks as any) + expect(messages).toHaveLength(1) + expect(addedMessages).toHaveLength(1) + }) +}) + +describe('openItemPreviewAction', () => { + // The action's `type` is the key the sessions page registers its handler under, + // so it must stay 'open_item_preview'; `previewKind`/`path` are passed verbatim + // to previewTargetForSessionTarget. + it('carries the kind and path through to the dispatch action', () => { + expect(openItemPreviewAction('flow', 'f/team/etl')).toEqual({ + id: 'open-item-preview:flow:f/team/etl', + type: 'open_item_preview', + label: 'Open flow preview', + previewKind: 'flow', + path: 'f/team/etl' + }) + }) + + // raw_app is the internal kind; the user-facing label says "app". + it('labels raw_app as "app"', () => { + expect(openItemPreviewAction('raw_app', 'u/me/dash').label).toBe('Open app preview') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 197f0d018f..53e72ab1c1 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -18,6 +18,8 @@ export const SPECIAL_MODULE_IDS = { } as const import { get } from 'svelte/store' import type { PasteAttachment } from './pasteTokens' +import { dataUrlToImagePart, type AttachedImage } from './imageUtils' +import type { AttachedTextFile } from './textFileUtils' import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } from './context' import { workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' @@ -38,6 +40,7 @@ import { } from '$lib/gen' import uFuzzy from '@leeoniya/ufuzzy' import { emptyString } from '$lib/utils' +import { logFeatureUsage } from '$lib/utils/featureUsage' import { forLater } from '$lib/forLater' import { scriptLangToEditorLang } from '$lib/scripts' import { getCurrentModel } from '$lib/aiStore' @@ -468,6 +471,13 @@ export type UserDisplayMessage = BaseDisplayMessage & { // Collapsed big-paste blobs referenced by tokens in `content`. Lets the // bubble render/expand chips; the LLM message stores the expanded text. pastes?: PasteAttachment[] + // Images the user attached to this message (drag/drop/paste), rendered as + // thumbnails in the bubble. The LLM message carries them as image_url parts. + images?: AttachedImage[] + // Text files the user attached to this message, rendered as chips in the + // bubble. The prompt lists them by reference; the content here is the durable + // copy, re-registered into the session file store on load for tool reads. + files?: AttachedTextFile[] } export type CreatedResourceTriggerKind = @@ -477,6 +487,7 @@ export type CreatedResourceTriggerKind = | 'nats' | 'postgres' | 'mqtt' + | 'amqp' | 'sqs' | 'gcp' | 'azure' @@ -504,7 +515,34 @@ export type NavigateAction = { page: string } -export type ToolDisplayAction = CreatedResourceAction | NavigateAction +/** Kinds of previewable item a write tool can land — the subset of draft item + * kinds a session preview can host. */ +export type PreviewCardKind = 'script' | 'flow' | 'raw_app' + +// A discrete card shown on a tool call that created or updated a workspace item. +// Clicking it opens the item's live preview in the session side panel — or focuses +// the tab if it is already open. The handler is registered by the sessions page +// (the only surface with a preview panel). +export type OpenItemPreviewAction = { + id: string + type: 'open_item_preview' + label: string + previewKind: PreviewCardKind + path: string +} + +export type ToolDisplayAction = CreatedResourceAction | NavigateAction | OpenItemPreviewAction + +/** Build the action a preview card dispatches from its (kind, path). */ +export function openItemPreviewAction(kind: PreviewCardKind, path: string): OpenItemPreviewAction { + return { + id: `open-item-preview:${kind}:${path}`, + type: 'open_item_preview', + label: `Open ${kind === 'raw_app' ? 'app' : kind} preview`, + previewKind: kind, + path + } +} export type UserQuestionDisplay = { question: string @@ -522,6 +560,12 @@ export function answeredChoices(q: UserQuestionDisplay): string[] | undefined { return q.selectedChoices ?? (q.selectedChoice ? [q.selectedChoice] : undefined) } +/** One page hit from a provider-side web search (OpenAI sources carry no title). */ +export type WebSearchSource = { + url: string + title?: string +} + export type ToolDisplayMessage = { role: 'tool' tool_call_id: string @@ -539,6 +583,13 @@ export type ToolDisplayMessage = { showFade?: boolean actions?: ToolDisplayAction[] userQuestion?: UserQuestionDisplay + webSearchSources?: WebSearchSource[] + /** Data URL of an image the tool produced (e.g. take_screenshot), shown on the card. */ + imageUrl?: string + /** Workspace item this tool created or updated. Rendered as a discrete, + * always-visible card that opens (or focuses) the item's preview in the + * session side panel. Set only for session chats — the side panel is their surface. */ + previewCard?: { kind: PreviewCardKind; path: string } } export type AssistantDisplayMessage = BaseDisplayMessage & { @@ -556,13 +607,20 @@ export type AssistantDisplayMessage = BaseDisplayMessage & { /** * Compaction boundary: replaces the summarized prefix in BOTH displayMessages - * and the API messages (where it is a plain user message). It carries no index - * because it is never a restart target — only the surviving tail's user - * messages are rewound to. + * and the API messages (where it is a plain user message). It is never a restart + * target — only the surviving tail's user messages are rewound to. */ export type SummaryDisplayMessage = { role: 'summary' content: string + // Index of the summary's API message, tracked ONLY so orphan detection can tell + // when a later drop-oldest compaction drops it (index goes negative) and its + // carried files must move to the roster. Not a restart target. Absent on + // summaries loaded from pre-existing history. + index?: number + // Files attached to messages the summary folded away — carried forward so + // they stay tool-readable (and reload-safe) after compaction. + files?: AttachedTextFile[] } export type DisplayMessage = @@ -629,6 +687,37 @@ async function callTool({ type MaybePromise = T | Promise +const MAX_TOOL_ERROR_LENGTH = 2000 + +/** ApiError from the generated client carries the server's message in `body`, + * not `message` — dig it out so tool failures show the real cause. Capped so a + * verbose error body (e.g. an HTML error page) can't flood the chat context. */ +export function formatToolError(error: any): string { + const bodyMessage = + error?.body?.error?.message ?? + error?.body?.message ?? + (typeof error?.body?.error === 'string' ? error.body.error : undefined) + const body = + bodyMessage ?? + (typeof error?.body === 'string' + ? error.body + : error?.body !== undefined + ? stringifyErrorBody(error.body) + : undefined) + const message = String(body || error?.message || error) + return message.length > MAX_TOOL_ERROR_LENGTH + ? message.slice(0, MAX_TOOL_ERROR_LENGTH) + '... (truncated)' + : message +} + +function stringifyErrorBody(body: unknown): string { + try { + return JSON.stringify(body) + } catch { + return String(body) + } +} + export async function processToolCall({ tools, toolCall, @@ -719,6 +808,11 @@ export async function processToolCall({ } let result = '' + // Key by the resolved tool's declared name, not the model-provided string, + // so hallucinated tool names never enter telemetry. + if (tool) { + logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId }) + } try { result = await callTool({ tools, @@ -735,17 +829,12 @@ export async function processToolCall({ }) } catch (err) { console.error(err) + const errorMessage = formatToolError(err) toolCallbacks.setToolStatus(toolCall.id, { isLoading: false, isStreamingArguments: false, - error: 'An error occurred while calling the tool' + error: errorMessage }) - const errorMessage = - typeof err === 'object' && 'message' in err - ? err.message - : typeof err === 'string' - ? err - : 'An error occurred while calling the tool' result = `Error while calling tool: ${errorMessage}` } const toAdd = { @@ -756,14 +845,46 @@ export async function processToolCall({ return toAdd } catch (err) { console.error(err) + const errorMessage = formatToolError(err) + toolCallbacks.setToolStatus(toolCall.id, { + isLoading: false, + isStreamingArguments: false, + error: errorMessage + }) return { role: 'tool' as const, tool_call_id: toolCall.id, - content: 'Error while calling tool' + content: `Error while calling tool: ${errorMessage}` } } } +/** + * Flush images buffered by tools during a batch (via toolCallbacks.attachToolImage) + * as ONE follow-up user message, appended to both `messages` (sent on later + * iterations) and `addedMessages` (committed to history). Call this once per + * completion, right after the whole tool loop — never mid-batch, so every tool_call + * id is already answered by its tool result before this non-tool message. The image + * parts ride the same `image_url` carrier that the provider converters translate. + */ +export function appendPendingToolImages( + messages: ChatCompletionMessageParam[], + addedMessages: ChatCompletionMessageParam[], + toolCallbacks: ToolCallbacks +): void { + const images = toolCallbacks.takePendingToolImages?.() ?? [] + if (images.length === 0) return + const message: ChatCompletionMessageParam = { + role: 'user', + content: [ + { type: 'text', text: 'Screenshot(s) of the app preview:' }, + ...images.map((img) => dataUrlToImagePart(img.dataUrl)) + ] + } + messages.push(message) + addedMessages.push(message) +} + export interface Tool { def: ChatCompletionFunctionTool fn: (p: { @@ -788,6 +909,9 @@ export interface Tool { autoCollapseDetails?: boolean streamArguments?: boolean showFade?: boolean + /** Header shown while the model is still streaming this call's arguments, + * before `fn` runs and sets a real status. Defaults to "Calling ...". */ + streamingLabel?: string } /** Status of a job the chat started and tracks in the jobs tray. Mirrors the @@ -825,6 +949,9 @@ export type ChatJob = { detached: boolean /** Notify-only: whether its completion has been surfaced to the model yet. */ reported: boolean + /** Whether the user saw its terminal status in the jobs popover. Reviewed + * outcomes stop driving the segment chip's status readout. Persisted. */ + reviewed?: boolean /** Trimmed snapshot of the last fetched Job (heavy fields stripped, see * `trimJob`), fed to `` so the tray badge matches the runs page * exactly. Always written together with `status` from the SAME job so the two @@ -904,6 +1031,16 @@ export interface ToolCallbacks { onItemDeployed?: (itemKind: UserDraftItemKind, storagePath: string, deployedPath: string) => void /** A tool discarded a draft: the chat's touch on the item is undone. */ onItemDiscarded?: (itemKind: UserDraftItemKind, storagePath: string) => void + /** + * Buffer an image a tool produced (e.g. take_screenshot). Tool results are + * string-only and OpenAI forbids images in tool messages, so buffered images are + * flushed as a follow-up user message once the whole tool batch is answered (see + * appendPendingToolImages) — appending mid-batch would leave sibling tool_call ids + * unanswered before a non-tool message. + */ + attachToolImage?: (toolId: string, image: AttachedImage) => void + /** Drain every image buffered this batch (insertion order), clearing the buffer. */ + takePendingToolImages?: () => AttachedImage[] } export function createToolDef( diff --git a/frontend/src/lib/components/copilot/chat/textFileUtils.test.ts b/frontend/src/lib/components/copilot/chat/textFileUtils.test.ts new file mode 100644 index 0000000000..aca305d2b9 Binary files /dev/null and b/frontend/src/lib/components/copilot/chat/textFileUtils.test.ts differ diff --git a/frontend/src/lib/components/copilot/chat/textFileUtils.ts b/frontend/src/lib/components/copilot/chat/textFileUtils.ts new file mode 100644 index 0000000000..a34d4999d6 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/textFileUtils.ts @@ -0,0 +1,208 @@ +/** + * Message-scoped text-file attachments for the GLOBAL chat composer. + * + * Text files attach like images: chips in the composer, riding the next + * message, cleared on send. Unlike images, only a *reference* (name + size) + * enters the prompt — the content is registered into the session file store at + * send and the model reads it on demand via the file tools, same as a DOM pick + * is inspected via the DOM tools. The full content stays on the message for + * the bubble preview and for re-registration on edit/retry. + */ +import { isTextFile } from './files/fileEngine' + +export type AttachedTextFile = { + name: string + /** Stable reference the transcript, prompt, and file tools join on — a content + * hash of (name, content), see attachedTextFileId. The name is display-only + * and may collide freely. Absent only on transcripts persisted before ids + * existed; hydrated (deterministically, from the same hash) on chat load. */ + id?: string + /** Original filename before a courtesy rename (set only when one happened). + * Lets duplicate detection recognize a re-drop of the same source file without + * inferring provenance from the display name — a user's real `report (2).md` + * must never be mistaken for a rename of `report.md`. */ + sourceName?: string + content: string +} + +/** + * Files one message may carry. Enforced wherever a message is assembled, not + * just at the composer: queuing clears the composer, so its own count would + * reset and let repeated sends stack an unbounded batch into a single message. + */ +export const MAX_ATTACHED_FILES = 8 + +/** + * Per-file byte cap. The model reads content on demand (never inlined), so + * this only bounds what rides the message state and the chat history's + * persisted snapshot — a sanity ceiling, not a context-window one. Larger + * files can be linked via their folder instead. + */ +export const MAX_TEXT_FILE_BYTES = 1_000_000 + +/** + * Cumulative cap across a conversation. Message-file content lives inside the + * transcript (DisplayMessage.files) and is rewritten with every history save, + * so without a conversation-level bound repeated attachments would grow the + * in-memory record and its IndexedDB copy without limit. Enforced at attach + * time against transcript + queue + composer bytes. + */ +export const MAX_CONVERSATION_FILE_BYTES = 5_000_000 + +/** Display names are rendered into model-facing prompt blocks, and OS + * filenames may legally contain control characters (even newlines on POSIX) — + * a crafted name must not be able to inject prompt structure. Applied at + * attach and again wherever a (possibly legacy) name is printed for the model. */ +export function sanitizeAttachmentName(name: string): string { + return name.replace(/[\u0000-\u001f\u007f]+/g, ' ').trim() || 'file' +} + +/** Read a file for message attachment. Returns null when the sniff says binary + * or the content exceeds MAX_TEXT_FILE_BYTES — the cap is enforced here at the + * reader, not only at callers' pre-checks, so no ingestion path can persist an + * oversized attachment (decoding can also grow past the raw size when malformed + * UTF-8 expands to replacement characters). + * The id is minted by the composer after name finalization (a same-name clash in + * one draft gets a courtesy rename first, and the id hashes the final name). */ +export async function fileToAttachedTextFile(file: File): Promise { + if (file.size > MAX_TEXT_FILE_BYTES) return null + if (!(await isTextFile(file))) return null + const content = await file.text() + if (textByteLength(content) > MAX_TEXT_FILE_BYTES) return null + return { name: sanitizeAttachmentName(file.name), content } +} + +/** Line count as the file tools report it (fileEngine.buildLineIndex): an empty + * file has 0 lines and a trailing newline is not an extra line. The prompt must + * advertise the same number or the model requests invalid read_file ranges. */ +export function textLineCount(content: string): number { + if (content === '') return 0 + return content.split('\n').length - (content.endsWith('\n') ? 1 : 0) +} + +/** Admit files in order while their DECODED byte size fits `budget`. Admission + * pre-checks use raw File.size, but the committed charge is the decoded UTF-8 + * length, which malformed input inflates (an invalid byte decodes to a 3-byte + * replacement character) — so the commit step must re-check against what will + * actually be charged. */ +export function admitWithinByteBudget( + files: AttachedTextFile[], + budget: number +): { admitted: AttachedTextFile[]; dropped: number } { + const admitted: AttachedTextFile[] = [] + let dropped = 0 + for (const f of files) { + const bytes = textByteLength(f.content) + if (bytes <= budget) { + admitted.push(f) + budget -= bytes + } else { + dropped++ + } + } + return { admitted, dropped } +} + +// cyrb53 (public-domain hash by bryc) — chosen over crypto.subtle because it is +// synchronous and works on plain-HTTP deployments where SubtleCrypto is absent. +function cyrb53(str: string, seed: number): number { + let h1 = 0xdeadbeef ^ seed + let h2 = 0x41c6ce57 ^ seed + for (let i = 0; i < str.length; i++) { + const ch = str.charCodeAt(i) + h1 = Math.imul(h1 ^ ch, 2654435761) + h2 = Math.imul(h2 ^ ch, 1597334677) + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909) + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909) + return 4294967296 * (2097151 & h2) + (h1 >>> 0) +} + +/** + * Deterministic content-hash id for a message attachment. Identity derived from + * the file itself: re-registration after reload/rollback lands on the same id by + * construction, identical attaches dedupe, and legacy transcripts hydrate their + * ids without migration state. Two seeded cyrb53 passes (~106 bits) — collision + * odds are negligible at conversation scale, and the context is not adversarial + * (a user's own attachments). + */ +export function attachedTextFileId(name: string, content: string): string { + // NUL separator: unambiguous split (filenames cannot contain it), so + // two (name, content) pairs never hash alike across the boundary. + const input = `${name}\u0000${content}` + return `f${cyrb53(input, 1).toString(36)}${cyrb53(input, 2).toString(36)}` +} + +/** Return `files` with every entry carrying its id (legacy rows hydrated). */ +export function withAttachedTextFileIds(files: AttachedTextFile[]): AttachedTextFile[] { + return files.map((f) => (f.id ? f : { ...f, id: attachedTextFileId(f.name, f.content) })) +} + +/** + * Fold freshly-read files into a draft's attachment list: identical + * (name, content) duplicates are dropped, same-name-different-content clashes + * get the courtesy rename, and ids are minted from the final name. Must run + * against the LIVE list in the synchronous commit step — attach batches overlap + * (each awaits its file reads), so dedupe/rename decisions made mid-read would + * be stale by commit time. + */ +export function foldIntoDraft( + current: AttachedTextFile[], + reads: { name: string; content: string; sourceName?: string }[] +): AttachedTextFile[] { + const commit: AttachedTextFile[] = [] + for (const f of reads) { + const draft = [...current, ...commit] + // Sanitized here, not assumed: the fold is the draft side's single choke + // point, so every rule a name needs is applied by this one call — a future + // entry point cannot skip one. + const readName = sanitizeAttachmentName(f.name) + // "Same file dropped twice" means same original (name, content) — a + // courtesy-renamed copy carries its original name in sourceName rather than + // inferring provenance from the display name (a user's real `report (2).md` + // is not a rename of `report.md`). Folds compose: a file renamed by an + // earlier fold (composer → queue → dequeue) keeps its original source. + const src = f.sourceName ? sanitizeAttachmentName(f.sourceName) : readName + if ( + draft.some( + (x) => x.content === f.content && (x.name === readName || (x.sourceName ?? x.name) === src) + ) + ) { + continue + } + const name = uniqueDraftFileName( + readName, + draft.map((x) => x.name) + ) + commit.push({ + name, + content: f.content, + id: attachedTextFileId(name, f.content), + ...(name !== src ? { sourceName: src } : {}) + }) + } + return commit +} + +/** Courtesy rename for a same-name clash within one message draft: `notes.md` → + * `notes (2).md`. Display-only — identity is the id, and names may collide + * across messages — but two identical labels inside one draft would be + * indistinguishable to the user and the model alike. */ +export function uniqueDraftFileName(original: string, taken: Iterable): string { + const names = new Set(taken) + if (!names.has(original)) return original + const dot = original.lastIndexOf('.') + const base = dot > 0 ? original.slice(0, dot) : original + const ext = dot > 0 ? original.slice(dot) : '' + let n = 2 + while (names.has(`${base} (${n})${ext}`)) n++ + return `${base} (${n})${ext}` +} + +/** UTF-8 byte length of attachment content — budget math must match the byte + * caps, and string length undercounts multibyte text. */ +export function textByteLength(content: string): number { + return new TextEncoder().encode(content).length +} diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.ts index 51290f9269..0091e11227 100644 --- a/frontend/src/lib/components/copilot/chat/tokenUsage.ts +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.ts @@ -70,12 +70,15 @@ export function anthropicUsageToChatTokenUsage( } } +// Unlike Anthropic, OpenAI's input_tokens already includes cached tokens +// (input_tokens_details.cached_tokens is a subset), so it must not be added again. export function openAIResponsesUsageToChatTokenUsage( usage: | { input_tokens?: number | null output_tokens?: number | null total_tokens?: number | null + input_tokens_details?: { cached_tokens?: number | null } | null } | null | undefined @@ -90,12 +93,15 @@ export function openAIResponsesUsageToChatTokenUsage( } } +// Unlike Anthropic, OpenAI's prompt_tokens already includes cached tokens +// (prompt_tokens_details.cached_tokens is a subset), so it must not be added again. export function openAICompletionsUsageToChatTokenUsage( usage: | { prompt_tokens?: number | null completion_tokens?: number | null total_tokens?: number | null + prompt_tokens_details?: { cached_tokens?: number | null } | null } | null | undefined diff --git a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts index 8a7932b8d9..b1a6f695d3 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceItems.svelte.ts @@ -7,6 +7,7 @@ import { HttpTriggerService, KafkaTriggerService, MqttTriggerService, + AmqpTriggerService, NatsTriggerService, PostgresTriggerService, ResourceService, @@ -35,6 +36,7 @@ export type WindmillItemKind = | 'nats_trigger' | 'postgres_trigger' | 'mqtt_trigger' + | 'amqp_trigger' | 'sqs_trigger' | 'gcp_trigger' | 'azure_trigger' @@ -123,6 +125,7 @@ const workspaceItemLoaders: Array<{ list: (workspace) => PostgresTriggerService.listPostgresTriggers({ workspace }) }, { kind: 'mqtt_trigger', list: (workspace) => MqttTriggerService.listMqttTriggers({ workspace }) }, + { kind: 'amqp_trigger', list: (workspace) => AmqpTriggerService.listAmqpTriggers({ workspace }) }, { kind: 'sqs_trigger', list: (workspace) => SqsTriggerService.listSqsTriggers({ workspace }) }, { kind: 'gcp_trigger', list: (workspace) => GcpTriggerService.listGcpTriggers({ workspace }) }, { diff --git a/frontend/src/lib/components/copilot/chat/workspaceTools.ts b/frontend/src/lib/components/copilot/chat/workspaceTools.ts index 5f069f1322..dabe63b5bc 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceTools.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceTools.ts @@ -1,19 +1,24 @@ import { AzureTriggerService, + EmailTriggerService, GcpTriggerService, HttpTriggerService, KafkaTriggerService, MqttTriggerService, + AmqpTriggerService, NatsTriggerService, PostgresTriggerService, ScheduleService, + SettingService, SqsTriggerService, WebsocketTriggerService, type AzureTriggerData, type GcpTriggerData, + type NewEmailTrigger, type NewHttpTrigger, type NewKafkaTrigger, type NewMqttTrigger, + type NewAmqpTrigger, type NewNatsTrigger, type NewPostgresTrigger, type NewSchedule, @@ -28,6 +33,7 @@ import { import { z } from 'zod' import { createToolDef, + formatToolError, type CreatedResourceTriggerKind, type Tool, type ToolCallbacks, @@ -44,9 +50,11 @@ type TriggerRequestByKind = { nats: NewNatsTrigger postgres: NewPostgresTrigger mqtt: NewMqttTrigger + amqp: NewAmqpTrigger sqs: NewSqsTrigger gcp: GcpTriggerData azure: AzureTriggerData + email: NewEmailTrigger } type TriggerRequestBody = TriggerRequestByKind[TriggerKind] @@ -113,7 +121,7 @@ const createScheduleToolDef = createToolDef( const createTriggerToolDef = createToolDef( createTriggerToolSchema, 'create_trigger', - 'Create a trigger for the current script or flow.', + 'Create a trigger for the current script or flow. For an email trigger (kind "email"), config.local_part is the local part of the receiving address (before the @); the tool reports the full address on success. Email triggers require email triggering to be configured on the instance — if it is not, the tool returns setup guidance instead of creating one.', { strict: false } ) @@ -154,6 +162,12 @@ const triggerConfigs = { create: (data: { workspace: string; requestBody: NewMqttTrigger }) => MqttTriggerService.createMqttTrigger(data) }, + amqp: { + label: 'AMQP trigger', + requestSchema: triggerRequestSchemas.amqp as z.ZodType, + create: (data: { workspace: string; requestBody: NewAmqpTrigger }) => + AmqpTriggerService.createAmqpTrigger(data) + }, sqs: { label: 'SQS trigger', requestSchema: triggerRequestSchemas.sqs as z.ZodType, @@ -171,6 +185,12 @@ const triggerConfigs = { requestSchema: triggerRequestSchemas.azure as z.ZodType, create: (data: { workspace: string; requestBody: AzureTriggerData }) => AzureTriggerService.createAzureTrigger(data) + }, + email: { + label: 'Email trigger', + requestSchema: triggerRequestSchemas.email as z.ZodType, + create: (data: { workspace: string; requestBody: NewEmailTrigger }) => + EmailTriggerService.createEmailTrigger(data) } } satisfies { [K in TriggerKind]: { @@ -237,30 +257,6 @@ function parseWithExplicitErrors(schema: z.ZodType, value: unknown, label: return result.data } -function formatApiError(error: any): string { - const bodyMessage = - error?.body?.error?.message ?? - error?.body?.message ?? - (typeof error?.body?.error === 'string' ? error.body.error : undefined) - const body = - bodyMessage ?? - (typeof error?.body === 'string' - ? error.body - : error?.body !== undefined - ? stringifyErrorBody(error.body) - : undefined) - const message = body || error?.message || String(error) - return error?.status ? `HTTP ${error.status}: ${message}` : message -} - -function stringifyErrorBody(body: unknown): string { - try { - return JSON.stringify(body) - } catch { - return String(body) - } -} - function setToolError(toolCallbacks: ToolCallbacks, toolId: string, error: unknown): string { const errorMessage = error instanceof Error ? error.message : String(error) toolCallbacks.setToolStatus(toolId, { @@ -302,7 +298,7 @@ const createScheduleTool: Tool = { } }) } catch (error) { - throw new Error(`Invalid schedule or timezone: ${formatApiError(error)}`) + throw new Error(`Invalid schedule or timezone: ${formatToolError(error)}`) } toolCallbacks.setToolStatus(toolId, { @@ -325,7 +321,9 @@ const createScheduleTool: Tool = { }) return JSON.stringify(toolResult) } catch (error) { - throw new Error(`Failed to create schedule "${requestBody.path}": ${formatApiError(error)}`) + throw new Error( + `Failed to create schedule "${requestBody.path}": ${formatToolError(error)}` + ) } } catch (error) { return setToolError(toolCallbacks, toolId, error) @@ -333,6 +331,39 @@ const createScheduleTool: Tool = { } } +const EMAIL_TRIGGER_DOCS = 'https://windmill.dev/docs/advanced/email_triggers' + +type EmailTriggerAvailability = + | { available: true; domain: string } + | { available: false; hint: string } + +/** + * Email triggers only work once an instance superadmin has stood up an SMTP + * server forwarding to Windmill and set the `email_domain` global setting + * (readable by any authed user). When it is unset the create call would fail + * opaquely, so we surface actionable, role-aware setup guidance instead. + */ +async function resolveEmailTriggerAvailability(): Promise { + let emailDomain: unknown + try { + emailDomain = await SettingService.getGlobal({ key: 'email_domain' }) + } catch { + emailDomain = undefined + } + if (typeof emailDomain === 'string' && emailDomain.trim() !== '') { + return { available: true, domain: emailDomain } + } + const [{ get }, { userStore }] = await Promise.all([ + import('svelte/store'), + import('$lib/stores') + ]) + const isSuperadmin = get(userStore)?.is_super_admin ?? false + const hint = isSuperadmin + ? `Email triggering is not set up on this instance yet, so no email trigger was created. As a superadmin, enable it: run an SMTP server that forwards inbound mail to Windmill and set the "email_domain" instance setting (Instance settings). See ${EMAIL_TRIGGER_DOCS}. Once configured, ask again and I will create the trigger.` + : `Email triggering is not set up on this instance yet, so no email trigger was created. Ask an instance superadmin to enable it: they need to run an SMTP server that forwards inbound mail to Windmill and set the "email_domain" instance setting. See ${EMAIL_TRIGGER_DOCS}. Once it is configured, ask again and I will create the trigger.` + return { available: false, hint } +} + const createTriggerTool: Tool = { def: createTriggerToolDef, requiresConfirmation: true, @@ -354,22 +385,52 @@ const createTriggerTool: Tool = { triggerConfig.label ) + let emailDomain: string | undefined + if (parsedArgs.kind === 'email') { + // `workspaced_local_part` maps to a NOT NULL column; the model may omit it, + // so default it here before the request is sent, not just when formatting the address. + const emailBody = requestBody as NewEmailTrigger + emailBody.workspaced_local_part = emailBody.workspaced_local_part ?? false + const availability = await resolveEmailTriggerAvailability() + if (!availability.available) { + toolCallbacks.setToolStatus(toolId, { + content: availability.hint, + isLoading: false, + needsConfirmation: false + }) + return availability.hint + } + emailDomain = availability.domain + } + toolCallbacks.setToolStatus(toolId, { content: `Creating ${triggerConfig.label} "${requestBody.path}"...` }) try { const result = await triggerConfig.create({ workspace, requestBody } as never) const targetKind = getActionTargetKind(requestBody.is_flow) + const emailAddress = + parsedArgs.kind === 'email' && emailDomain !== undefined + ? (await import('$lib/components/triggers/email/utils')).getEmailAddress( + (requestBody as NewEmailTrigger).local_part, + (requestBody as NewEmailTrigger).workspaced_local_part ?? false, + workspace, + emailDomain + ) + : undefined const toolResult = { success: true, kind: parsedArgs.kind, path: requestBody.path, target_path: requestBody.script_path, target_kind: targetKind, - backend_result: result + backend_result: result, + ...(emailAddress ? { email_address: emailAddress } : {}) } toolCallbacks.setToolStatus(toolId, { - content: `Created ${triggerConfig.label} "${requestBody.path}"`, + content: emailAddress + ? `Created ${triggerConfig.label} "${requestBody.path}" (send email to ${emailAddress})` + : `Created ${triggerConfig.label} "${requestBody.path}"`, result: toolResult, actions: [ createOpenTriggerAction( @@ -383,7 +444,7 @@ const createTriggerTool: Tool = { return JSON.stringify(toolResult) } catch (error) { throw new Error( - `Failed to create ${triggerConfig.label} "${requestBody.path}": ${formatApiError(error)}` + `Failed to create ${triggerConfig.label} "${requestBody.path}": ${formatToolError(error)}` ) } } catch (error) { diff --git a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts index d989412bb1..6aa40bce75 100644 --- a/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts +++ b/frontend/src/lib/components/copilot/chat/workspaceToolsZod.gen.ts @@ -293,6 +293,43 @@ export const mqttTriggerRequestSchema = z.object({ "labels": z.array(z.string()).optional() }) +export const amqpTriggerRequestSchema = z.object({ + "amqp_resource_path": z.string().describe("Path to the AMQP resource containing broker connection configuration"), + "queue_name": z.string().describe("Name of the queue to consume messages from"), + "exchange": z.object({ + "exchange_name": z.string().describe("Name of the exchange to bind the consumed queue to"), + "routing_keys": z.array(z.string()).describe("Routing keys used to bind the queue to the exchange").optional() + }).describe("Optional exchange binding for the consumed queue").nullable().optional(), + "options": z.object({ + "declare_queue": z.boolean().describe("Declare the queue (durable) before consuming; when false the queue is declared passively and must already exist").optional(), + "prefetch_count": z.number().int().gte(1).lte(65535).describe("Maximum number of unacknowledged messages the broker delivers at once (1-65535)").optional() + }).describe("Optional consumer options (queue declaration, prefetch)").nullable().optional(), + "path": z.string().describe("The unique Windmill path for this trigger. Must be of the form `u//` or `f//`."), + "script_path": z.string().describe("Path to the script or flow to execute when a message is received"), + "is_flow": z.boolean().describe("True if script_path points to a flow, false if it points to a script"), + "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), + "error_handler_path": z.string().describe("Path to a script or flow to run when the triggered job fails").optional(), + "error_handler_args": z.record(z.string(), z.any()).describe("Arguments to pass to the error handler").optional(), + "retry": z.object({ + "constant": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "seconds": z.number().int().describe("Seconds to wait between retries").optional() + }).describe("Retry with constant delay between attempts").optional(), + "exponential": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), + "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), + "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() + }).describe("Retry with exponential backoff (delay doubles each time)").optional(), + "retry_if": z.object({ + "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") + }).describe("Conditional retry based on error or result").optional() + }).describe("Retry configuration for failed executions").optional(), + "permissioned_as": z.string().describe("The user or group this trigger runs as. Used during deployment to preserve the original trigger owner.").optional(), + "preserve_permissioned_as": z.boolean().describe("When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it.").optional(), + "labels": z.array(z.string()).optional() +}) + export const sqsTriggerRequestSchema = z.object({ "queue_url": z.string().describe("The full URL of the AWS SQS queue to poll for messages"), "aws_auth_resource_type": z.enum(["oidc", "credentials"]).describe("Authentication type - 'credentials' for access key/secret, 'oidc' for OpenID Connect"), @@ -397,6 +434,35 @@ export const azureTriggerRequestSchema = z.object({ "labels": z.array(z.string()).optional() }).describe("Data for creating or updating an Azure Event Grid trigger.") +export const emailTriggerRequestSchema = z.object({ + "path": z.string(), + "script_path": z.string(), + "local_part": z.string(), + "workspaced_local_part": z.boolean().optional(), + "is_flow": z.boolean(), + "error_handler_path": z.string().optional(), + "error_handler_args": z.record(z.string(), z.any()).describe("The arguments to pass to the script or flow").optional(), + "retry": z.object({ + "constant": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "seconds": z.number().int().describe("Seconds to wait between retries").optional() + }).describe("Retry with constant delay between attempts").optional(), + "exponential": z.object({ + "attempts": z.number().int().describe("Number of retry attempts").optional(), + "multiplier": z.number().int().describe("Multiplier for exponential backoff").optional(), + "seconds": z.number().int().gte(1).describe("Initial delay in seconds").optional(), + "random_factor": z.number().int().gte(0).lte(100).describe("Random jitter percentage (0-100) to avoid thundering herd").optional() + }).describe("Retry with exponential backoff (delay doubles each time)").optional(), + "retry_if": z.object({ + "expr": z.string().describe("JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables") + }).describe("Conditional retry based on error or result").optional() + }).describe("Retry configuration for failed module executions").optional(), + "mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(), + "permissioned_as": z.string().describe("The user or group this trigger runs as. Used during deployment to preserve the original trigger owner.").optional(), + "preserve_permissioned_as": z.boolean().describe("When true and the caller is a member of the 'wm_deployers' group, preserves the original permissioned_as value instead of overwriting it.").optional(), + "labels": z.array(z.string()).optional() +}) + export const variableRequestSchema = z.object({ "path": z.string().describe("The path to the variable"), "value": z.string().describe("The value of the variable"), @@ -425,9 +491,11 @@ export const triggerRequestSchemas = { nats: natsTriggerRequestSchema, postgres: postgresTriggerRequestSchema, mqtt: mqttTriggerRequestSchema, + amqp: amqpTriggerRequestSchema, sqs: sqsTriggerRequestSchema, gcp: gcpTriggerRequestSchema, azure: azureTriggerRequestSchema, + email: emailTriggerRequestSchema, } as const const triggerPathSchema = z.string().min(1).describe("The unique Windmill path for this trigger. Must be of the form `u//` or `f//`. This is the trigger object path, not the HTTP route path.") @@ -440,9 +508,11 @@ export const createTriggerToolSchema = z.object({ "nats", "postgres", "mqtt", + "amqp", "sqs", "gcp", "azure", + "email", ]), path: triggerPathSchema, config: z.union([ @@ -452,8 +522,10 @@ export const createTriggerToolSchema = z.object({ natsTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), postgresTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), mqttTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), + amqpTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), sqsTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), gcpTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), azureTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), + emailTriggerRequestSchema.omit({ path: true, script_path: true, is_flow: true }), ]) }) diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index ccc8b85a65..17a7ad6508 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -14,6 +14,7 @@ import { parseFimCompletionChoice } from './fim' import { getKnownModelContextWindow, getModelContextWindow, + modelSupportsVision, requiresMaxCompletionTokens } from './modelConfig' import { supportsAutocomplete } from './utils' @@ -255,3 +256,40 @@ describe('model context windows', () => { expect(getModelContextWindow('some-custom-model')).toBe(128000) }) }) + +describe('modelSupportsVision', () => { + // One listed pair pins the lookup mechanism (exact pair match, case-insensitive + // model ids); the set's contents are data, not behavior. + it('refuses images on a listed provider:model pair', () => { + expect(modelSupportsVision('groq' as any, 'llama-3.3-70b-versatile')).toBe(false) + expect(modelSupportsVision('azure_foundry' as any, 'DeepSeek-R1')).toBe(false) + }) + + // The reason this is an exact-match set. Each of these WOULD be wrongly blocked + // by a substring of an entry above, and each takes images via its API. + it.each([ + ['azure_foundry', 'Mistral-Large-3'], // substring of 'mistral-large-2411' + ['azure_foundry', 'Phi-4-multimodal-instruct'], // substring of 'phi-4' + ['openrouter', 'meta-llama/llama-3.2-90b-vision-instruct'], // 'llama-3.2-...' + ['groq', 'meta-llama/llama-4-scout-17b-16e-instruct'], + ['groq', 'qwen/qwen3.6-27b'] + ])('does not let a text-only id shadow the vision model %s/%s', (provider, model) => { + expect(modelSupportsVision(provider as any, model)).toBe(true) + }) + + // Permissive by design: a wrong "no" blocks a working model with no override, + // while a wrong "yes" costs one turn and the failure path recovers it. + it('allows unknown and custom models', () => { + expect(modelSupportsVision('customai' as any, 'some-internal-vlm')).toBe(true) + expect(modelSupportsVision('deepseek' as any, 'deepseek-v9-sees-everything')).toBe(true) + expect(modelSupportsVision(undefined, undefined)).toBe(true) + }) + + // The reason entries are keyed by provider, not id alone: an id proves nothing + // about a different endpoint. A Custom AI deployment serving a vision model + // under a colliding name must not inherit another provider's verdict. + it("does not apply one provider's text-only verdict to another provider's model", () => { + expect(modelSupportsVision('customai' as any, 'deepseek-chat')).toBe(true) + expect(modelSupportsVision('customai' as any, 'llama-3.3-70b-versatile')).toBe(true) + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 88afa58b7d..ed36c8e42d 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -17,7 +17,12 @@ import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { requiresMaxCompletionTokens, usesAnthropicMessagesApi } from './modelConfig' import { applyReasoningToConfig } from './reasoningRegistry' import { formatResourceTypes } from './utils' -import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' +import { + appendPendingToolImages, + processToolCall, + type Tool, + type ToolCallbacks +} from './chat/shared' import { hasValidToolCallArguments } from './chat/toolCallArguments' import { getNonStreamingOpenAIResponsesCompletion, @@ -487,7 +492,9 @@ export async function testKey({ // getNonStreamingCompletion routes Anthropic-Messages-API models (native // Anthropic and Claude on Azure Foundry) through the Anthropic SDK and // everything else through OpenAI chat completions, so the test exercises the - // same request shape the feature actually sends. + // same request shape the feature actually sends. The cap keeps max_tokens + // under the Anthropic SDK's non-streaming pre-flight limit (~21k tokens), + // which would otherwise reject the request before it is sent. await getNonStreamingCompletion(messages, abortController, { apiKey, workspace, @@ -495,7 +502,8 @@ export async function testKey({ forceModelProvider: { model: modelToTest, provider: aiProvider - } + }, + maxTokensCap: METADATA_MAX_TOKENS }) } @@ -1176,7 +1184,7 @@ export async function parseOpenAICompletion( // Display tool call with streaming parameters if enabled callbacks.setToolStatus(toolCallId, { isLoading: true, - content: `Calling ${funcName}...`, + content: tool?.streamingLabel ?? `Calling ${funcName}...`, toolName: funcName, isStreamingArguments: shouldStream, showFade: tool?.showFade, @@ -1257,6 +1265,7 @@ export async function parseOpenAICompletion( messages.push(messageToAdd) addedMessages.push(messageToAdd) } + appendPendingToolImages(messages, addedMessages, callbacks) } else if (malformedFunctionCallError) { // Malformed function call with no tool calls - create artificial tool call to inform AI const fakeToolCallId = generateRandomString() diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index af0dd942b2..62e5a3542a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -72,3 +72,69 @@ export function getModelContextWindow(model: string) { // Trim/compaction logic needs a number; assume a conservative window when unknown. return getKnownModelContextWindow(model) ?? 128000 } + +/** + * Best-effort check that a model can accept image input. There is no per-model vision + * metadata in the codebase, so this is deliberately permissive: it returns true unless + * the model is a known text-only one that would 400 on an image part. Used to gate the + * image-attach affordance and the screenshot follow-up; when unsure it allows the image + * (the user explicitly attached it — better to try than to silently drop it). + */ +export function modelSupportsVision( + provider: AIProvider | undefined, + model: string | undefined +): boolean { + if (!provider) return true + return !TEXT_ONLY_MODELS.has(`${provider}:${(model ?? '').toLowerCase()}`) +} + +/** + * Models whose provider API refuses image content, matched by exact + * `provider:model` pair — not by id alone, because an id proves nothing about a + * different endpoint (a Custom AI deployment may serve a vision model under a + * name that collides with someone's text-only id, and there is no override). + * + * The question is not whether a model can see, but whether its provider's API + * accepts image parts — the two diverge, and the divergence is invisible from a + * name: DeepSeek V4 ships vision in its chat UI while its API has no image + * content type, and o3-mini gained vision in ChatGPT that the API never exposed. + * So this is a cache of one provider's API surface at one moment, and it rots. + * Wrong entries are asymmetric: a missing one costs a single turn and + * self-corrects (the request fails, the image is dropped, the user is told), + * while a wrong one blocks a working model with no override. Hence exact pairs + * only, and only where a provider doc says so. + * + * Substrings are specifically avoided: `mistral-large` would also match + * Mistral Large 3, which does take images, and `phi-4` would match + * Phi-4-multimodal, which does too. + */ +const TEXT_ONLY_MODELS = new Set([ + 'openai:o1-mini', + 'openai:o3-mini', + 'azure_openai:o1-mini', + 'azure_openai:o3-mini', + 'mistral:codestral-latest', + // deepseek — vision exists in their chat product, not in the API + 'deepseek:deepseek-v4-pro', + 'deepseek:deepseek-v4-flash', + 'deepseek:deepseek-chat', + 'deepseek:deepseek-reasoner', + 'groq:llama-3.3-70b-versatile', + 'groq:llama-3.1-8b-instant', + // gpt-oss (text-only everywhere it is hosted) — on groq it succeeds the two + // llama defaults above, which retire 2026-08-16 + 'groq:openai/gpt-oss-120b', + 'groq:openai/gpt-oss-20b', + 'openrouter:openai/gpt-oss-120b', + 'openrouter:openai/gpt-oss-20b', + 'togetherai:openai/gpt-oss-120b', + 'togetherai:openai/gpt-oss-20b', + // azure_foundry serves DeepSeek-V4-Pro under the same id as deepseek's API + 'azure_foundry:deepseek-v4-pro', + 'azure_foundry:deepseek-r1', + 'azure_foundry:llama-3.3-70b-instruct', + 'azure_foundry:phi-4', + 'azure_foundry:mistral-large-2411', + 'openrouter:meta-llama/llama-3.2-3b-instruct:free', + 'togetherai:meta-llama/llama-3.3-70b-instruct-turbo' +]) diff --git a/frontend/src/lib/components/custom_ui.ts b/frontend/src/lib/components/custom_ui.ts index a04c9ea4b5..041c2dd23a 100644 --- a/frontend/src/lib/components/custom_ui.ts +++ b/frontend/src/lib/components/custom_ui.ts @@ -3,6 +3,7 @@ import type { SupportedLanguage } from '$lib/common' export type FlowBuilderWhitelabelCustomUi = { topBar?: { path?: boolean + editablePath?: boolean export?: boolean history?: boolean aiBuilder?: boolean @@ -121,6 +122,7 @@ export type EditorBarUi = { s3object?: boolean database?: boolean ducklake?: boolean + metrics?: boolean dataTable?: boolean debug?: boolean history?: boolean diff --git a/frontend/src/lib/components/datatableSchemaSql.ts b/frontend/src/lib/components/datatableSchemaSql.ts new file mode 100644 index 0000000000..cd3d0f9154 --- /dev/null +++ b/frontend/src/lib/components/datatableSchemaSql.ts @@ -0,0 +1,220 @@ +import type { + TableEditorValues, + TableEditorValuesColumn, + TableEditorForeignKey +} from '$lib/components/apps/components/display/dbtable/tableEditor' +import { + diffTableEditorValues, + type AlterTableValues, + makeAlterTableQueries +} from '$lib/components/apps/components/display/dbtable/queries/alterTable' +import { renderForeignKey } from '$lib/components/apps/components/display/dbtable/queries/dbQueriesUtils' +import type { GetDatatableFullSchemaResponse } from '$lib/gen' + +export type DatabaseSchema = Record> + +export function apiSchemaToEditorSchema(apiSchema: GetDatatableFullSchemaResponse): DatabaseSchema { + const result: DatabaseSchema = {} + for (const [schemaName, tables] of Object.entries(apiSchema)) { + result[schemaName] = {} + for (const [tableName, table] of Object.entries(tables as Record)) { + if (!table || typeof table !== 'object') continue + result[schemaName][tableName] = { + name: table.name ?? tableName, + columns: (table.columns ?? []).map( + (c: any): TableEditorValuesColumn => ({ + name: c.name, + datatype: c.datatype, + primaryKey: c.primary_key ?? c.primaryKey, + defaultValue: c.default_value ?? c.defaultValue, + nullable: c.nullable + }) + ), + foreignKeys: (table.foreign_keys ?? table.foreignKeys ?? []).map( + (fk: any): TableEditorForeignKey => ({ + targetTable: fk.target_table ?? fk.targetTable, + columns: (fk.columns ?? []).map((col: any) => ({ + sourceColumn: col.source_column ?? col.sourceColumn, + targetColumn: col.target_column ?? col.targetColumn + })), + onDelete: (fk.on_delete ?? fk.onDelete ?? 'NO ACTION') as + | 'CASCADE' + | 'SET NULL' + | 'NO ACTION', + onUpdate: (fk.on_update ?? fk.onUpdate ?? 'NO ACTION') as + | 'CASCADE' + | 'SET NULL' + | 'NO ACTION', + fk_constraint_name: fk.fk_constraint_name + }) + ), + pk_constraint_name: table.pk_constraint_name + } + } + } + return result +} + +export type TableDiff = { + schemaName: string + tableName: string + kind: 'added' | 'removed' | 'modified' + operations?: AlterTableValues +} + +export type DatatableDiff = { + datatableName: string + aheadChanges: TableDiff[] + behindChanges: TableDiff[] + originalSchema: DatabaseSchema + parentSchema: DatabaseSchema + forkSchema: DatabaseSchema +} + +export function diffDatabaseSchemas( + original: DatabaseSchema, + current: DatabaseSchema +): TableDiff[] { + const diffs: TableDiff[] = [] + const allSchemas = new Set([...Object.keys(original), ...Object.keys(current)]) + for (const schemaName of allSchemas) { + const origTables = original[schemaName] ?? {} + const currTables = current[schemaName] ?? {} + const allTables = new Set([...Object.keys(origTables), ...Object.keys(currTables)]) + for (const tableName of allTables) { + const origTable = origTables[tableName] + const currTable = currTables[tableName] + if (!origTable && currTable) { + diffs.push({ schemaName, tableName, kind: 'added' }) + } else if (origTable && !currTable) { + diffs.push({ schemaName, tableName, kind: 'removed' }) + } else if (origTable && currTable) { + const currWithInitial: TableEditorValues = { + ...currTable, + columns: currTable.columns.map((col) => ({ + ...col, + initialName: col.name, + defaultValue: col.defaultValue ? `{${col.defaultValue}}` : undefined + })) + } + const origTableTransformed: TableEditorValues = { + ...origTable, + columns: origTable.columns.map((col) => ({ + ...col, + defaultValue: col.defaultValue ? `{${col.defaultValue}}` : undefined + })) + } + const diff = diffTableEditorValues(origTableTransformed, currWithInitial) + if (diff.operations.length > 0) { + diffs.push({ schemaName, tableName, kind: 'modified', operations: diff }) + } + } + } + } + return diffs +} + +export function computeDatatableDiff( + datatableName: string, + originalSchema: DatabaseSchema, + parentSchema: DatabaseSchema, + forkSchema: DatabaseSchema +): DatatableDiff { + return { + datatableName, + behindChanges: diffDatabaseSchemas(originalSchema, parentSchema), + aheadChanges: diffDatabaseSchemas(originalSchema, forkSchema), + originalSchema, + parentSchema, + forkSchema + } +} + +/** Detect PostgreSQL auto-increment columns and return the serial type + cleaned props. + * e.g. bigint + nextval('seq'::regclass) → BIGSERIAL (no DEFAULT needed) */ +function resolveColumnType(c: TableEditorValuesColumn): { + datatype: string + defaultValue: string | undefined +} { + const dv = c.defaultValue ?? '' + if (/^{?nextval\(/.test(dv)) { + const dt = c.datatype?.toLowerCase() ?? '' + if (dt === 'bigint') return { datatype: 'BIGSERIAL', defaultValue: undefined } + if (dt === 'integer' || dt === 'int') return { datatype: 'SERIAL', defaultValue: undefined } + if (dt === 'smallint') return { datatype: 'SMALLSERIAL', defaultValue: undefined } + } + return { datatype: c.datatype, defaultValue: c.defaultValue } +} + +/** + * SQL for an added table, with the CREATE TABLE and the FK constraints split so + * callers creating several tables can emit every CREATE before any constraint — + * required for circular FKs, where no creation order satisfies inline FKs. + */ +export function generateAddedTableSql( + change: TableDiff, + sourceSchema: DatabaseSchema, + options?: { ifNotExists?: boolean } +): { create: string; constraints: string[] } | undefined { + const table = sourceSchema[change.schemaName]?.[change.tableName] + if (!table) return undefined + const colDefs = table.columns + .map((c) => { + const { datatype, defaultValue } = resolveColumnType(c) + let def = `"${c.name}" ${datatype}` + if (c.nullable === false) def += ' NOT NULL' + if (defaultValue) def += ` DEFAULT ${defaultValue}` + return def + }) + .join(',\n ') + const pkCols = table.columns.filter((c) => c.primaryKey).map((c) => `"${c.name}"`) + const pkLine = pkCols.length > 0 ? `,\n PRIMARY KEY (${pkCols.join(', ')})` : '' + const qualifiedName = `"${change.schemaName}"."${change.tableName}"` + const createKeyword = options?.ifNotExists ? 'CREATE TABLE IF NOT EXISTS' : 'CREATE TABLE' + // The target may not have the schema at all (fresh data table import). + const schemaDdl = + change.schemaName !== 'public' ? `CREATE SCHEMA IF NOT EXISTS "${change.schemaName}";\n` : '' + const create = `${schemaDdl}${createKeyword} ${qualifiedName} (\n ${colDefs}${pkLine}\n);` + const constraints: string[] = [] + for (const fk of table.foreignKeys ?? []) { + const fkSql = renderForeignKey(fk, { + useSchema: true, + dbType: 'postgresql', + tableName: change.tableName + }) + // With IF NOT EXISTS the table may pre-exist with this FK already in + // place; an unconditional ADD would then abort the whole transaction. + // The constraint name is emitted unquoted, so Postgres folds it to + // lowercase — compare against the folded form. + const fkName = options?.ifNotExists + ? fkSql.match(/^CONSTRAINT\s+(\S+)/)?.[1]?.toLowerCase() + : undefined + constraints.push( + fkName + ? `DO $$\nBEGIN\n IF NOT EXISTS (\n SELECT 1 FROM pg_constraint\n WHERE conname = '${fkName}' AND conrelid = '${qualifiedName}'::regclass\n ) THEN\n ALTER TABLE ${qualifiedName} ADD ${fkSql};\n END IF;\nEND $$;` + : `ALTER TABLE ${qualifiedName} ADD ${fkSql};` + ) + } + return { create, constraints } +} + +export function generateMigrationSql( + change: TableDiff, + sourceSchema: DatabaseSchema, + options?: { ifNotExists?: boolean } +): string { + if (change.kind === 'modified' && change.operations) { + const queries = makeAlterTableQueries(change.operations, 'postgresql', change.schemaName) + if (queries.length === 0) return '' + return 'BEGIN;\n' + queries.join('\n') + '\nCOMMIT;' + } + if (change.kind === 'added') { + const gen = generateAddedTableSql(change, sourceSchema, options) + if (!gen) return '' + return `BEGIN;\n${[gen.create, ...gen.constraints].join('\n')}\nCOMMIT;` + } + if (change.kind === 'removed') { + return `BEGIN;\nDROP TABLE IF EXISTS "${change.schemaName}"."${change.tableName}";\nCOMMIT;` + } + return '' +} diff --git a/frontend/src/lib/components/drillPicker.ts b/frontend/src/lib/components/drillPicker.ts index 5f3b5d6a25..71e13742e0 100644 --- a/frontend/src/lib/components/drillPicker.ts +++ b/frontend/src/lib/components/drillPicker.ts @@ -21,6 +21,10 @@ export type DrillLeaf = { /** Optional override for the fuzzy-search haystack. Defaults to * `label` (or `secondary` when label is empty). */ searchableText?: string + /** Category header this leaf renders under, in both the browse list and + * search results (a `searchGroup` branch ancestor wins in search). + * Consecutive leaves sharing a section share one header. */ + section?: string /** Marks this leaf as the user's current location — gets `aria-current` * and a styled, no-op click. */ current?: boolean diff --git a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte index 3f070dd74e..dc2cba93d4 100644 --- a/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputsQuick.svelte @@ -146,7 +146,7 @@ let inlineScripts: [string, SupportedLanguage | 'docker'][] = $state([]) - const enterpriseLangs = ['bigquery', 'snowflake', 'mssql', 'oracledb'] + const enterpriseLangs = ['mssql', 'oracledb'] function computeInlineScriptChoices( funcDesc: string, diff --git a/frontend/src/lib/components/flows/content/FlowModuleCache.svelte b/frontend/src/lib/components/flows/content/FlowModuleCache.svelte index f9744af36c..99fc4c7b3a 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleCache.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleCache.svelte @@ -6,12 +6,29 @@ import type { FlowModule } from '$lib/gen' import { SecondsInput } from '../../common' + import WorkspaceScriptSettingInfo from './WorkspaceScriptSettingInfo.svelte' interface Props { flowModule: FlowModule + // For workspace-script steps: the cache_ttl currently set on the referenced + // script, and a shortcut to edit it. Undefined for inline/subflow steps. + workspaceScriptCacheTtl?: number | undefined + loadingWorkspaceScript?: boolean + workspaceScriptError?: string | undefined + canEditWorkspaceScript?: boolean + workspaceScriptNoEditReason?: string | undefined + onEditWorkspaceScript?: () => void } - let { flowModule = $bindable() }: Props = $props() + let { + flowModule = $bindable(), + workspaceScriptCacheTtl = undefined, + loadingWorkspaceScript = false, + workspaceScriptError = undefined, + canEditWorkspaceScript = false, + workspaceScriptNoEditReason = undefined, + onEditWorkspaceScript + }: Props = $props() let isCacheEnabled = $derived(Boolean(flowModule.cache_ttl)) @@ -25,10 +42,22 @@ {/snippet} - {#if flowModule.value.type != 'rawscript'} + {#if flowModule.value.type == 'script'} + + {:else if flowModule.value.type != 'rawscript'}

- The cache settings need to be set in the referenced script/flow settings directly. Cache for - hub scripts is not available yet. + The cache settings need to be set in the referenced flow settings directly.

{:else} ('FlowEditorContext') const selectedId = $derived(selectionManager.getSelectedId()) @@ -180,6 +185,59 @@ let assets = $derived((flowModule.value.type === 'rawscript' && flowModule.value.assets) || []) const flowGraphAssetsCtx = getContext('FlowGraphAssetContext') + // For workspace-script steps, load the referenced script's advanced settings so + // the delegating settings tabs (concurrency, cache, ...) can show current values + // and offer an "Edit script settings" shortcut instead of a bare warning. + const referencedScriptSettings = useWorkspaceScriptSettings( + () => (flowModule.value.type === 'script' ? flowModule.value.path : undefined), + () => (flowModule.value.type === 'script' ? flowModule.value.hash : undefined), + () => opWs + ) + // Hub scripts, hash-pinned steps, and embeddings that disable script editing + // can't have their settings edited from here. The drawer must also be mounted: + // local-dev editors (Dev.svelte / flows/dev) provide the context store but never + // render the drawer, so editing there would be a no-op — keep values read-only. + let canEditWorkspaceScriptSettings = $derived( + flowModule.value.type === 'script' && + !flowModule.value.path?.startsWith('hub/') && + flowModule.value.hash == undefined && + customUi?.scriptEdit != false && + $workspaceScriptSettingsDrawer != undefined + ) + let workspaceScriptNoEditReason = $derived( + flowModule.value.type !== 'script' || canEditWorkspaceScriptSettings + ? undefined + : flowModule.value.path?.startsWith('hub/') + ? 'Hub scripts cannot be edited from here.' + : flowModule.value.hash != undefined + ? 'Steps pinned to a specific version cannot be edited from here.' + : 'Editing script settings is not available in this editor.' + ) + // Non-positive concurrent_limit / cache_ttl are treated as unset by the runtime (legacy rows). + let referencedConcurrentLimit = $derived( + referencedScriptSettings.settings?.concurrent_limit != undefined && + referencedScriptSettings.settings.concurrent_limit > 0 + ? referencedScriptSettings.settings.concurrent_limit + : undefined + ) + let referencedCacheTtl = $derived( + referencedScriptSettings.settings?.cache_ttl != undefined && + referencedScriptSettings.settings.cache_ttl > 0 + ? referencedScriptSettings.settings.cache_ttl + : undefined + ) + function openWorkspaceScriptSettings() { + if (flowModule.value.type !== 'script') return + $workspaceScriptSettingsDrawer?.openDrawer( + flowModule.value.path, + flowModule.value.hash, + async () => { + await referencedScriptSettings.reload() + forceReload++ + } + ) + } + // UI Intent handling for AI tool control useUiIntent(`flow-${flowModule.id}`, { openTab: (tab) => { @@ -770,6 +828,9 @@ flowModule.value.hash = await getLatestHashForScript(flowModule.value.path, opWs) } forceReload++ + // Keep the surfaced concurrency/cache values and badges in sync after + // a settings/code save from the header (path/hash may be unchanged). + await referencedScriptSettings.reload() await reload(flowModule) } if (flowModule.value.type == 'flow') { @@ -991,6 +1052,16 @@ {:else if flowModule.value.type === 'script'} {#if !noEditor && (customUi?.hubCode != false || !flowModule?.value?.path?.startsWith('hub/'))}
+ {#if referencedScriptSettings.settings && getActiveScriptSettingsBadges(referencedScriptSettings.settings).length > 0} +
+ +
+ {/if} {#key forceReload} + {:else if flowModule.value.type == 'script'} + {:else} - The concurrency limit of a workspace script is only settable in the - script metadata itself. For hub scripts, this feature is non available - yet. + The concurrency limit of a referenced flow is only settable in the + flow settings directly. {/if} @@ -1322,7 +1412,15 @@
{:else if advancedSelected === 'cache'}
- +
{:else if advancedSelected === 'early-stop'} diff --git a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte index c3d6742b54..b34a78e1db 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleHeader.svelte @@ -13,7 +13,8 @@ Repeat, Square, Pin, - Save + Save, + Settings } from 'lucide-svelte' import Popover from '../../Popover.svelte' import type { FlowEditorContext } from '../types' @@ -28,7 +29,7 @@ } let { module, tag }: Props = $props() - const { scriptEditorDrawer, flowEditorDrawer, opWorkspace } = + const { scriptEditorDrawer, workspaceScriptSettingsDrawer, flowEditorDrawer, opWorkspace } = getContext('FlowEditorContext') const dispatch = createEventDispatcher() @@ -107,26 +108,54 @@ {/if} {#if module.value.type === 'script'} {#if !module.value.path.startsWith('hub/') && customUi?.scriptEdit != false} - + + + + + {/snippet} + + diff --git a/frontend/src/lib/components/flows/content/WorkspaceScriptSettingInfo.svelte b/frontend/src/lib/components/flows/content/WorkspaceScriptSettingInfo.svelte new file mode 100644 index 0000000000..505474c24c --- /dev/null +++ b/frontend/src/lib/components/flows/content/WorkspaceScriptSettingInfo.svelte @@ -0,0 +1,61 @@ + + +
+
+ + {label} is managed on the referenced workspace script. + + {#if canEdit} + + {/if} +
+
+ {#if loading} + + Loading current value… + + {:else if error} + Could not load the current value: {error} + {:else if active} + {valueText} + {:else} + Not set on the script. + {/if} +
+ {#if !canEdit && noEditReason} + {noEditReason} + {/if} +
diff --git a/frontend/src/lib/components/flows/content/WorkspaceScriptSettingsDrawer.svelte b/frontend/src/lib/components/flows/content/WorkspaceScriptSettingsDrawer.svelte new file mode 100644 index 0000000000..3a88e6214a --- /dev/null +++ b/frontend/src/lib/components/flows/content/WorkspaceScriptSettingsDrawer.svelte @@ -0,0 +1,150 @@ + + + + drawer?.closeDrawer()}> + {#if loading} +
+ + Loading +
+ {:else if loadError || !script} +
+ + {loadError ?? 'Script not found.'} + + {#if current} + + {/if} +
+ {:else} +
+
+ {script.path} + +
+

+ Saving creates a new version of the workspace script with these runtime settings. The code + is left unchanged. +

+ +
+ {/if} + {#snippet actions()} + + {/snippet} +
+
diff --git a/frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte b/frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte index 573db39435..0484762705 100644 --- a/frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte +++ b/frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte @@ -14,7 +14,7 @@ let { disabled = false, label, lang = undefined, id = undefined }: Props = $props() - const enterpriseLangs = ['bigquery', 'snowflake', 'mssql', 'oracledb'] + const enterpriseLangs = ['mssql', 'oracledb'] diff --git a/frontend/src/lib/components/flows/types.ts b/frontend/src/lib/components/flows/types.ts index 41165865ae..a527bd765d 100644 --- a/frontend/src/lib/components/flows/types.ts +++ b/frontend/src/lib/components/flows/types.ts @@ -2,6 +2,7 @@ import type { Job, OpenFlow } from '$lib/gen' import type { History } from '$lib/history.svelte' import type { Writable } from 'svelte/store' import type ScriptEditorDrawer from './content/ScriptEditorDrawer.svelte' +import type WorkspaceScriptSettingsDrawer from './content/WorkspaceScriptSettingsDrawer.svelte' import type FlowEditorDrawer from './content/FlowEditorDrawer.svelte' import type { FlowState } from './flowState' import type { FlowBuilderWhitelabelCustomUi } from '../custom_ui' @@ -76,6 +77,7 @@ export type FlowEditorContext = { currentEditor: Writable previewArgs: StateStore> scriptEditorDrawer: Writable + workspaceScriptSettingsDrawer: Writable flowEditorDrawer: Writable history: History pathStore: Writable diff --git a/frontend/src/lib/components/flows/useWorkspaceScriptSettings.svelte.ts b/frontend/src/lib/components/flows/useWorkspaceScriptSettings.svelte.ts new file mode 100644 index 0000000000..861f85e7bf --- /dev/null +++ b/frontend/src/lib/components/flows/useWorkspaceScriptSettings.svelte.ts @@ -0,0 +1,72 @@ +import { ScriptService } from '$lib/gen' +import type { ScriptAdvancedSettingsFields } from '$lib/components/scriptSettings' + +// Loads the advanced runtime settings (concurrency, cache, timeout, ...) of the +// workspace script referenced by a flow step, so the flow editor can surface the +// current values instead of only a "set it on the script" warning. Reactive to +// the path/hash/workspace getters; call reload() after saving new settings. +export function useWorkspaceScriptSettings( + pathGetter: () => string | undefined, + hashGetter: () => string | undefined, + workspaceGetter: () => string | undefined +) { + let settings = $state(undefined) + let loading = $state(false) + let error = $state(undefined) + // Guards against an older in-flight load resolving after a newer one and + // clobbering the displayed settings when path/hash change quickly. + let loadSeq = 0 + + async function load( + path: string | undefined, + hash: string | undefined, + workspace: string | undefined + ) { + const seq = ++loadSeq + if (!path || !workspace || path.startsWith('hub/')) { + settings = undefined + error = undefined + // Clear here too: this supersedes any in-flight load, whose guarded + // finally can no longer reset loading, else the card spins forever. + loading = false + return + } + loading = true + error = undefined + try { + const script = hash + ? await ScriptService.getScriptByHash({ workspace, hash }) + : await ScriptService.getScriptByPath({ workspace, path }) + if (seq !== loadSeq) return + settings = script as ScriptAdvancedSettingsFields + } catch (e) { + console.error('Could not load referenced script settings', e) + if (seq === loadSeq) { + settings = undefined + // Surface failure so cards distinguish "load failed" from "not set". + error = `${(e as { body?: string })?.body ?? e}` + } + } finally { + if (seq === loadSeq) loading = false + } + } + + $effect(() => { + load(pathGetter(), hashGetter(), workspaceGetter()) + }) + + return { + get settings() { + return settings + }, + get loading() { + return loading + }, + get error() { + return error + }, + reload() { + return load(pathGetter(), hashGetter(), workspaceGetter()) + } + } +} diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index f17cdd23d4..c69245b0e5 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -45,7 +45,7 @@ export type GitSyncSettings = { export type ModalState = { push: { idx: number; repo: GitSyncRepository; open: boolean } | null pull: { idx: number; repo: GitSyncRepository; open: boolean; settingsOnly?: boolean } | null - success: { open: boolean; savedWithoutInit?: boolean } | null + success: { open: boolean; savedWithoutInit?: boolean; autoPullOn?: boolean } | null } export type ValidationState = { @@ -179,7 +179,10 @@ export function createGitSyncContext(workspace: string) { use_individual_branch: repo.use_individual_branch, group_by_folder: repo.group_by_folder, settings: repo.settings, - exclude_types_override: repo.exclude_types_override + exclude_types_override: repo.exclude_types_override, + auto_pull: repo.auto_pull, + promotion_open_prs: repo.promotion_open_prs, + fork_open_prs: repo.fork_open_prs } } @@ -198,7 +201,13 @@ export function createGitSyncContext(workspace: string) { exclude_types_override: [], legacyImported: false, isUnsavedConnection: true, - collapsed: false + collapsed: false, + // New connections default to pulling changes from Git (webhook with a + // polling fallback), forks included. Existing repos load without + // auto_pull and stay off until an admin opts in, so upgrades never + // start auto-deploying. + auto_pull: { enabled: true, mode: 'auto', sync_forks: true }, + fork_open_prs: true }) gitSyncTestJobs.push({ jobId: '', @@ -273,8 +282,8 @@ export function createGitSyncContext(workspace: string) { closeModal('pull') } - function showSuccessModal(savedWithoutInit?: boolean) { - activeModals.success = { open: true, savedWithoutInit } + function showSuccessModal(savedWithoutInit?: boolean, autoPullOn?: boolean) { + activeModals.success = { open: true, savedWithoutInit, autoPullOn } } function closeSuccessModal() { @@ -501,7 +510,10 @@ export function createGitSyncContext(workspace: string) { use_individual_branch: repoToSave.use_individual_branch, group_by_folder: repoToSave.group_by_folder, settings: repoToSave.settings, - exclude_types_override: repoToSave.exclude_types_override + exclude_types_override: repoToSave.exclude_types_override, + auto_pull: repoToSave.auto_pull, + promotion_open_prs: repoToSave.promotion_open_prs, + fork_open_prs: repoToSave.fork_open_prs } } }) @@ -516,7 +528,7 @@ export function createGitSyncContext(workspace: string) { repoToSave.detectionState = undefined repoToSave.extractedSettings = undefined // Show success modal for new connections - showSuccessModal(savedWithoutInit) + showSuccessModal(savedWithoutInit, repoToSave.auto_pull?.enabled === true) } } @@ -672,6 +684,9 @@ export function createGitSyncContext(workspace: string) { legacyImported: false, isUnsavedConnection: true, collapsed: false + // Pull-from-Git defaults are applied by the repository card once the + // selected resource resolves: only app-backed repos (instant webhook + // delivery) default to auto-pull on; polling is opt-in for token repos. }) gitSyncTestJobs.push({ jobId: '', @@ -698,7 +713,10 @@ export function createGitSyncContext(workspace: string) { exclude_types_override: [], legacyImported: false, isUnsavedConnection: true, - collapsed: false + collapsed: false, + // New promotion repos default to opening the PR in-app when a deploy + // pushes its wm_deploy/** branch (app-backed repos only at runtime). + promotion_open_prs: true }) gitSyncTestJobs.push({ jobId: '', diff --git a/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte b/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte index 15e23f7e1e..05925d6e26 100644 --- a/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte @@ -104,5 +104,6 @@ {/if} diff --git a/frontend/src/lib/components/git_sync/GitSyncModeDisplay.svelte b/frontend/src/lib/components/git_sync/GitSyncModeDisplay.svelte index 94c77a5974..214a87a01f 100644 --- a/frontend/src/lib/components/git_sync/GitSyncModeDisplay.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncModeDisplay.svelte @@ -1,25 +1,37 @@ -
- {#if mode === 'promotion'} -
Promotion: Creating branches whose promotion target is {targetBranch? `'${targetBranch}'` : - "the repo's default branch"}
- {#if repository?.group_by_folder} -
Grouped by folder
- {/if} - {:else if targetBranch} -
Sync: Syncing back to branch '{targetBranch}'
- {:else} -
Sync: Syncing back to the repo's default branch
+
+ {#if active} + {/if} + + {#if mode === 'promotion'} + On deploy, changes are pushed to a wm_deploy/… + branch{#if repository?.group_by_folder} + (grouped by folder){/if}; merging it into {#if targetBranch}{targetBranch}{:else}the repo's default branch{/if} promotes the change. + {:else} + Matching changes are committed to {#if targetBranch}{targetBranch}{:else}the repo's default branch{/if} on every deploy. + {/if} +
diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index ba5dbf4a03..30a8c3ef47 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -17,9 +17,11 @@ import DetectionFlow from './DetectionFlow.svelte' import { sendUserToast } from '$lib/toast' import { fade } from 'svelte/transition' - import { workspaceStore } from '$lib/stores' + import { workspaceStore, userWorkspaces, enterpriseLicense } from '$lib/stores' import type { GitSyncRepository } from './GitSyncContext.svelte' import GitSyncModeDisplay from './GitSyncModeDisplay.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import EEOnly from '$lib/components/EEOnly.svelte' import { ResourceService, VariableService } from '$lib/gen' let { @@ -31,7 +33,8 @@ repository = null, onAdd = null, isCollapsible = true, - showEmptyState = false + showEmptyState = false, + devPromotion = false } = $props<{ idx?: number | null isSecondary?: boolean @@ -42,6 +45,9 @@ onAdd?: (() => void) | null isCollapsible?: boolean showEmptyState?: boolean + // Dev workspace: this is the single inherited repo, and promotion is a + // toggle on it (reuse prod's repo) rather than a separately-configured one. + devPromotion?: boolean }>() const gitSyncContext = getGitSyncContext() @@ -49,9 +55,98 @@ const validation = $derived(idx !== null ? gitSyncContext.getValidation(idx) : null) const gitSyncTestJob = $derived(idx !== null ? gitSyncContext.gitSyncTestJobs?.[idx] : null) let confirmingDelete = $state(false) + + // Enable/disable automatic repo → workspace pulls, managing the optional + // auto_pull object without binding into a possibly-undefined value. + function setAutoPullEnabled(enabled: boolean) { + if (!repo) return + if (enabled) { + repo.auto_pull = { + ...(repo.auto_pull ?? {}), + enabled: true, + mode: repo.auto_pull?.mode ?? 'auto', + sync_forks: repo.auto_pull?.sync_forks ?? true + } + } else if (repo.auto_pull) { + repo.auto_pull = { ...repo.auto_pull, enabled: false } + } + } + + // Parent-level fork auto-sync (phase 5). Configured on the parent workspace's + // repo and applied to all of its forks, so hide it when the current workspace + // is itself a fork. + const currentWorkspaceData = $derived($userWorkspaces?.find((w) => w.id === $workspaceStore)) + // A fork or dev workspace: has a parent, or carries the wm-fork- id prefix + // (which survives if the parent is deleted). Mirrors the backend/CLI rule. + const isFork = $derived( + ($workspaceStore?.startsWith('wm-fork-') ?? false) || + !!currentWorkspaceData?.parent_workspace_id + ) + // A dev workspace is a fork that DOES run promotion mode (per-item + // wm_deploy/** branches into its parent), unlike a throwaway fork. + const isDevWorkspace = $derived(!!currentWorkspaceData?.is_dev_workspace) + function setSyncForks(v: boolean) { + if (repo?.auto_pull) repo.auto_pull = { ...repo.auto_pull, sync_forks: v } + } + function setForkOpenPrs(v: boolean) { + if (repo) repo.fork_open_prs = v + } + function setPromotionOpenPrs(v: boolean) { + if (repo) repo.promotion_open_prs = v + } + // The promotion toggles persist immediately and must not overlap: concurrent + // whole-repository saves can complete out of order (enabling runs extra + // backend checks), letting a stale earlier state overwrite the latest one. + let savingDevPromotion = $state(false) + async function setDevPromotion(v: boolean) { + if (!repo || idx === null || savingDevPromotion) return + const prevIndiv = repo.use_individual_branch + const prevGbf = repo.group_by_folder + repo.use_individual_branch = v + if (!v) repo.group_by_folder = false + savingDevPromotion = true + try { + await gitSyncContext.saveRepository(idx) + } catch (e) { + // The backend rejects promotion mode without an active EE plan; revert + // the optimistic toggle instead of leaving it stuck on until reload. + if (repo) { + repo.use_individual_branch = prevIndiv + repo.group_by_folder = prevGbf + } + sendUserToast(`Could not ${v ? 'enable' : 'disable'} Git promotion: ${e}`, true) + } finally { + savingDevPromotion = false + } + } + async function setGroupByFolder(v: boolean) { + if (!repo || idx === null || savingDevPromotion) return + const prev = repo.group_by_folder + repo.group_by_folder = v + savingDevPromotion = true + try { + await gitSyncContext.saveRepository(idx) + } catch (e) { + if (repo) repo.group_by_folder = prev + sendUserToast(`Could not change promotion granularity: ${e}`, true) + } finally { + savingDevPromotion = false + } + } + let targetBranch = $state(undefined) // Default to main, will be updated when resource is available + // The branch this fork workspace syncs with, mirroring the CLI/hub-script + // naming: a dev workspace uses its environment-label branch verbatim + // (dev/staging); a wm-fork- throwaway fork keeps only the slug. + const forkBranch = $derived( + currentWorkspaceData?.is_dev_workspace + ? (currentWorkspaceData?.dev_workspace_label ?? 'dev') + : `wm-fork/${targetBranch ?? 'main'}/${($workspaceStore ?? '').replace(/^wm-fork-/, '')}` + ) let resourceInfo = $state<{ url?: string; error?: string } | null>(null) let loadingResourceInfo = $state(false) + // Only GitHub App-backed repos can register webhooks; PAT repos poll only. + let isGithubApp = $state(false) // Update target branch when repository changes $effect(() => { @@ -82,9 +177,12 @@ const abortController = new AbortController() async function loadResourceInfo() { - if (repo?.git_repo_resource_path && !repo.isUnsavedConnection && $workspaceStore) { + if (repo?.git_repo_resource_path && $workspaceStore) { loadingResourceInfo = true resourceInfo = null + // Clear stale app state up front so a resource change or a failed + // fetch can't leave webhook/fork controls showing for the wrong repo. + isGithubApp = false try { const resource = await ResourceService.getResource({ workspace: $workspaceStore, @@ -94,6 +192,39 @@ if (!abortController.signal.aborted && resource?.value) { // Extract git URL from resource value const value = resource.value as Record + isGithubApp = value?.is_github_app === true + // A newly added sync connection defaults to pulling from Git only + // when the repository is app-backed (instant webhook delivery). + // Polling is opt-in for token repositories, and fork/dev workspaces + // never get the parent-only defaults (the backend rejects them). + // EE-only. + if ( + repoMode === 'sync' && + repo.isUnsavedConnection && + isGithubApp && + !isFork && + $enterpriseLicense && + repo.auto_pull === undefined + ) { + repo.auto_pull = { enabled: true, mode: 'auto', sync_forks: true } + } + // Promotion deploys push wm_deploy/** branches that exist to be + // merged; without a PR the deploy is an orphaned branch. Default + // the managed PR on where Windmill can open it (app-backed). + // Fork PRs stay opt-in everywhere. + if ( + repoMode === 'promotion' && + repo.isUnsavedConnection && + isGithubApp && + $enterpriseLicense && + repo.promotion_open_prs === undefined + ) { + repo.promotion_open_prs = true + } + // Webhook with polling fallback is the only delivery for app repos. + if (isGithubApp && repo.auto_pull?.mode === 'polling') { + repo.auto_pull = { ...repo.auto_pull, mode: 'auto' } + } let gitUrl = value?.url || value?.git_url if (gitUrl && typeof gitUrl === 'string') { @@ -156,6 +287,7 @@ } } else { resourceInfo = null + isGithubApp = false } } @@ -202,9 +334,9 @@ const displayDescription = $derived( variant === 'primary-sync' || variant === 'primary-promotion' ? mode === 'sync' - ? `Changes will be committed directly to the ${targetOrDefaultBranch} branch` + ? `Deploys are committed to the ${targetOrDefaultBranch} branch, and new commits to it can deploy back into this workspace automatically` : mode === 'promotion' - ? `Changes will be made to new branches whose promotion target is the ${targetOrDefaultBranch} branch of the repo to promote to. Remember to also setup Git Sync between the promotion workspace and repo for changes to be deployed when these branches are merged.` + ? `Each deploy in this workspace pushes its changes to a dedicated wm_deploy/** branch of the repository instead of committing to ${targetOrDefaultBranch} directly. Merging that branch into ${targetOrDefaultBranch} promotes the change: the workspace that syncs ${targetOrDefaultBranch} deploys it on merge, so set up Git Sync there. Windmill can open the pull request for each deploy branch (toggle below), or use the open-pr-on-commit workflow.` : null : null ) @@ -371,7 +503,7 @@ {#if !emptyString(repo.git_repo_resource_path)} +
+ {#if devPromotion && !repo.isUnsavedConnection} +
+ setDevPromotion(e.detail)} + /> + {#if repo.use_individual_branch} +
+ setGroupByFolder(e.detail)} + /> +
+ {/if} +
+ {/if} + {#if repoMode === 'promotion' && isGithubApp} +
+ + setPromotionOpenPrs(e.detail)} + /> +
+ {:else if repoMode === 'promotion' && !repo.isUnsavedConnection} +
+ To open a pull request for each deploy branch, set up the + open-pr-on-commit + workflow in the repository. Recommended: connect the repository through the + GitHub App and Windmill opens them automatically. +
+ {/if} + {#if repoMode === 'sync' && isFork} +
+ Deploys from this workspace are pushed to the + {forkBranch} branch of the shared repository, not to + the tracked branch. +
+ {:else if repoMode === 'sync' && !isFork} +
+ These push settings also apply to forks of this workspace: an item deployed in a + fork is pushed to the fork's own + wm-fork/… branch instead of the tracked branch. +
+ {#if isGithubApp} +
+ setForkOpenPrs(e.detail)} + > + {#snippet right()} + {#if !$enterpriseLicense}{/if} + {/snippet} + +
+ {:else} +
+ To open pull requests when an item is deployed in a fork, set up the + open-pr-on-fork-commit + workflow in the repository. Recommended: connect the repository through the + GitHub App and Windmill opens them automatically. +
+ {/if} + {/if} + {#if repo.open_pr_error} +
+ + {repo.open_pr_error} If this mentions permissions, the GitHub App installation may + not have approved the pull-request permission yet. + +
+ {/if} - - {#if !emptyString(repo.git_repo_resource_path) && !repo.legacyImported} -
-
Manual workspace content sync
-
+ + {#if repoMode === 'sync'} +
+
+
Pull from Git (Git → Windmill)
-
+ {#if isFork} + +
+ Automatic sync from Git is managed in the parent workspace's git sync settings. + When enabled there, changes to this fork's + {forkBranch} branch deploy here automatically. +
+ {#if repo.auto_pull?.last_pull_status} +
+ {#if repo.auto_pull.last_pull_status.success} + Last synced{repo.auto_pull.last_pull_status.synced_sha + ? ` to ${repo.auto_pull.last_pull_status.synced_sha.slice(0, 7)}` + : ''}. + {:else} + + Last sync failed{repo.auto_pull.last_pull_status.error + ? `: ${repo.auto_pull.last_pull_status.error}` + : ''}. + + {/if} +
+ {/if} + {:else} + setAutoPullEnabled(e.detail)} + > + {#snippet right()} + {#if !$enterpriseLicense}{/if} + {/snippet} + +
+ setSyncForks(e.detail)} + /> +
+ {/if} + {#if !isGithubApp && !loadingResourceInfo} +
+ + Pull for this repository checks the tracked branch about every minute; longer + gaps make drift and merge conflicts more likely. For instant pull, connect the + repository through the + GitHub App + (which also lets Windmill manage pull requests), or push changes into Windmill + with the + sync GitHub workflow. If you already push changes with a GitHub Action, keep either the Action or + automatic pull, not both, so they don't fight over deploys. + +
+ {/if} + {#if repo.auto_pull?.enabled} + {@const viaWebhook = repo.auto_pull?.webhook_id != null} + {#if isGithubApp} +
+ + If you previously set up a GitHub Action to push changes into Windmill, + remove it now so the two don't fight over deploys. + +
+ {/if} +
+ {#if repo.auto_pull?.last_pull_status} + {#if repo.auto_pull.last_pull_status.success} + Last synced{repo.auto_pull.last_pull_status.synced_sha + ? ` to ${repo.auto_pull.last_pull_status.synced_sha.slice(0, 7)}` + : ''}. + {viaWebhook + ? ' Syncing instantly via webhook.' + : ' Checking the tracked branch about every minute.'} + {:else} + + Last sync failed{repo.auto_pull.last_pull_status.error + ? `: ${repo.auto_pull.last_pull_status.error}` + : ''}. + + {/if} + {:else} + {viaWebhook + ? 'Connected via webhook. New commits to the tracked branch deploy instantly.' + : 'Checking the tracked branch about every minute. New commits deploy automatically.'} + {/if} +
+ {#if isGithubApp && repo.auto_pull?.webhook_error} +
+ + {repo.auto_pull.webhook_error} + +
+ {/if} + {/if}
{/if} -
+ {/if} {/if} {/if} {:else} @@ -580,7 +952,14 @@
{#if displayDescription} -

{displayDescription}

+

{displayDescription} + {#if mode === 'promotion'} + Learn more about Git Promotion + {/if}

{/if} {@render repositoryContent()} diff --git a/frontend/src/lib/components/git_sync/GitSyncSection.svelte b/frontend/src/lib/components/git_sync/GitSyncSection.svelte index 8041c734da..7327912697 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSection.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSection.svelte @@ -5,7 +5,8 @@ import { setGitSyncContext } from './GitSyncContext.svelte' import GitSyncRepositoryCard from './GitSyncRepositoryCard.svelte' import GitSyncModalManager from './GitSyncModalManager.svelte' - import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { enterpriseLicense, workspaceStore, userWorkspaces } from '$lib/stores' + import { base } from '$lib/base' import { WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { untrack } from 'svelte' @@ -42,6 +43,17 @@ const gitSyncAllowed = $derived(gitSyncStatus.enabled) const isFreeTier = $derived(gitSyncAllowed && !$enterpriseLicense) + // Throwaway forks never run promotion mode: their deploys always go to the + // fork's own wm-fork/** branch, so a promotion repo could never take effect + // (the backend rejects it too). A dev workspace is the exception — it deploys + // per-item wm_deploy/** branches that promote into its parent. Mirrors the + // backend/CLI rule. + const currentWorkspace = $derived($userWorkspaces?.find((w) => w.id === $workspaceStore)) + const isFork = $derived( + ($workspaceStore?.startsWith('wm-fork-') ?? false) || !!currentWorkspace?.parent_workspace_id + ) + const isDevWorkspace = $derived(!!currentWorkspace?.is_dev_workspace) + const showPromotion = $derived(!isFork || isDevWorkspace) const hasConfiguredRepos = $derived( gitSyncContext?.repositories?.some((r) => r.git_repo_resource_path) ?? false ) @@ -63,8 +75,34 @@ // Derived state for repository categorization const primarySync = $derived(gitSyncContext?.getPrimarySyncRepository() || null) const primaryPromotion = $derived(gitSyncContext?.getPrimaryPromotionRepository() || null) + // A dev workspace reuses the single repo it inherited from prod: whether it's + // currently in sync or promotion mode, it's the same one card, toggled between + // the two — so a dev never configures a separate promotion repo. + const devPrimaryRepo = $derived(isDevWorkspace ? (primarySync ?? primaryPromotion) : null) + // The single-repo dev UX (one card + promotion toggle, secondaries hidden) is + // only safe when the dev actually has one repo — i.e. it inherited prod's on + // fork. An ATTACHED dev keeps its own repos: with more than one, fall back to + // the normal layout so none are hidden and we don't present an unrelated repo + // as prod's promotion target. + const devSingleRepo = $derived(isDevWorkspace && (gitSyncContext?.repositories?.length ?? 0) <= 1) const secondarySync = $derived(gitSyncContext?.getSecondarySyncRepositories() || []) const secondaryPromotion = $derived(gitSyncContext?.getSecondaryPromotionRepositories() || []) + // Fork creation keeps only sync-mode repositories and a fork is refused a + // promotion one, so the single way a fork holds one is a dev workspace + // detached back into a plain fork. Deploys still sync through it (only the + // promotion branching is dropped), so name it instead of showing nothing. + const promotionModeRepos = $derived( + showPromotion ? [] : [primaryPromotion, ...secondaryPromotion].filter((r) => r != null) + ) + // Promotion is what a dev workspace does, so a fork that wants it can be + // re-designated as one. Pairing is prod-scoped and admin-gated there, so this + // only links to the parent's screen, and only when the parent is a workspace + // the user actually has. + const devPairingHref = $derived.by(() => { + const parent = currentWorkspace?.parent_workspace_id + if (!parent || !$userWorkspaces?.some((w) => w.id === parent)) return undefined + return `${base}/workspace_settings?workspace=${parent}&tab=dev_workspace` + }) // State for collapsible sections let secondarySyncExpanded = $state(false) @@ -88,7 +126,7 @@ {:else} {#snippet actions()} @@ -132,17 +170,18 @@
gitSyncContext.addSyncRepository()} isCollapsible={false} - showEmptyState={primarySync?.repo === null} + showEmptyState={(devSingleRepo ? devPrimaryRepo : primarySync)?.repo == null} + devPromotion={devSingleRepo && !!$enterpriseLicense} /> {#if $enterpriseLicense} - - {#if primarySync && !primarySync.repo?.isUnsavedConnection} + + {#if primarySync && !primarySync.repo?.isUnsavedConnection && !devSingleRepo} {#if secondarySync.length > 0 || secondarySyncExpanded}
- - {#if secondaryPromotionExpanded} -
- {#if secondaryPromotion.length === 0} -
- No secondary promotion repositories configured -
+ + {#if primaryPromotion && !primaryPromotion.repo?.isUnsavedConnection} + {#if secondaryPromotion.length > 0 || secondaryPromotionExpanded} +
+ - {#if !hasUnsavedSecondaryPromotion} -
- -
- {/if} + {#if secondaryPromotionExpanded} +
+ {#if secondaryPromotion.length === 0} +
+ No secondary promotion repositories configured +
+ {:else} + {#each secondaryPromotion as { repo, idx } (repo.git_repo_resource_path)} +
+ +
+ {/each} + {/if} + + {#if !hasUnsavedSecondaryPromotion} +
+ +
+ {/if} +
+ {/if} +
+ {:else} + + {#if !hasUnsavedSecondaryPromotion} +
+
{/if} -
- {:else} - - {#if !hasUnsavedSecondaryPromotion} -
- -
{/if} {/if} - {/if} -
+
+ {:else if !showPromotion} +
+ + Deploys in a fork always commit to the fork's own wm-fork/** branch, so a promotion + repository would never take effect here. Promote this fork's work by merging that + branch into the tracked branch instead. + {#if devPairingHref} +
+ To promote per item from this workspace, pair it with its parent as a + dev workspace. +
+ {/if} + {#if promotionModeRepos.length > 0} +
+ Still set to promotion mode here, and still syncing deploys to the fork's branch: + {promotionModeRepos.map((r) => r.repo.git_repo_resource_path).join(', ')} +
+ {/if} +
+
+ {/if} {/if} diff --git a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte index 5beddd05c6..32dc68a0f4 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte @@ -5,13 +5,10 @@ interface Props { open: boolean savedWithoutInit?: boolean + autoPullOn?: boolean } - let { - open = $bindable(false), - savedWithoutInit = false - }: Props = $props() - + let { open = $bindable(false), savedWithoutInit = false, autoPullOn = false }: Props = $props() @@ -23,7 +20,9 @@

Git sync connection saved successfully!

-

Your repository is now configured to receive changes from Windmill.

+

Your repository is now configured to receive changes from Windmill.

@@ -32,26 +31,39 @@

Repository saved without initialization

- Only new changes will be pushed to this repository. Existing content in Windmill has not been initialized to the repository. + Only new changes will be pushed to this repository. Existing content in Windmill has not + been initialized to the repository.

{/if} -
-

- - Optional: Enable automatic deployment from Git to Windmill -

-

- To automatically deploy changes from your Git repository back to Windmill (when PRs are merged), you can set up GitHub Actions or similar CI/CD workflows. -

+ {#if autoPullOn} +
+

+ + Pull from Git is on +

+

+ New commits to the tracked branch deploy into this workspace automatically. You can + adjust this anytime on the repository card. +

+
+ {:else} +
+

+ + Deploy changes from Git back to Windmill +

+

+ Turn on "Automatically deploy changes from Git" on the repository to have Windmill pull + new commits into this workspace for you. +

-

This setup enables:

-
    -
  • Automatic deployment to Windmill when PRs are merged
  • -
  • Full bidirectional sync between Git and Windmill
  • -
+

+ Prefer to control deployment from your own pipeline (tests, custom gating, deploy on PR + merge)? Set up GitHub Actions or similar CI/CD workflows instead. +

-
- + {/if}
diff --git a/frontend/src/lib/components/git_sync/PullWorkspaceModal.svelte b/frontend/src/lib/components/git_sync/PullWorkspaceModal.svelte index 37af944362..94845aa646 100644 --- a/frontend/src/lib/components/git_sync/PullWorkspaceModal.svelte +++ b/frontend/src/lib/components/git_sync/PullWorkspaceModal.svelte @@ -12,8 +12,8 @@ Edit3 } from 'lucide-svelte' import GitDiffPreview from '../GitDiffPreview.svelte' - import { JobService } from '$lib/gen' - import { workspaceStore } from '$lib/stores' + import { JobService, ResourceService } from '$lib/gen' + import { workspaceStore, userWorkspaces } from '$lib/stores' import { sendUserToast } from '$lib/toast' import hubPaths from '$lib/hubPaths.json' import { jobManager } from '$lib/services/JobManager' @@ -147,6 +147,29 @@ const workspace = $workspaceStore if (!workspace) return + // A dev workspace pulls from its environment-label branch (dev/staging) + // and a fork from its wm-fork// branch, not the resource's + // tracked branch. clone_ref falls back to the tracked branch in the + // pull script when the override branch doesn't exist yet. + const currentWs = $userWorkspaces?.find((w) => w.id === workspace) + const isFork = workspace.startsWith('wm-fork-') || Boolean(currentWs?.parent_workspace_id) + let cloneRef: string | undefined = undefined + if (currentWs?.is_dev_workspace) { + cloneRef = currentWs.dev_workspace_label ?? 'dev' + } else if (isFork) { + try { + const resource = await ResourceService.getResource({ + workspace, + path: gitRepoResourcePath + }) + const trackedBranch = (resource.value as any)?.branch + if (trackedBranch) { + cloneRef = `wm-fork/${trackedBranch}/${workspace.replace(/^wm-fork-/, '')}` + } + } catch (e) { + console.warn('Could not resolve tracked branch for fork pull:', e) + } + } const payload = { workspace_id: workspace, repo_url_resource_path: gitRepoResourcePath, @@ -155,7 +178,8 @@ only_wmill_yaml: settingsOnly, settings_json: JSON.stringify(uiState), use_promotion_overrides: - currentGitSyncSettings?.repositories?.[repoIndex!]?.use_individual_branch === true + currentGitSyncSettings?.repositories?.[repoIndex!]?.use_individual_branch === true, + ...(cloneRef ? { clone_ref: cloneRef } : {}) } const jobId = await JobService.runScriptByPath({ diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte index d7340e053a..ac41827d6c 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte @@ -5,7 +5,14 @@ import { getContext } from 'svelte' import { type TriggerContext } from '$lib/components/triggers' import { enterpriseLicense } from '$lib/stores' - import { MqttIcon, NatsIcon, KafkaIcon, AwsIcon, GoogleCloudIcon } from '$lib/components/icons' + import { + MqttIcon, + AmqpIcon, + NatsIcon, + KafkaIcon, + AwsIcon, + GoogleCloudIcon + } from '$lib/components/icons' import AzureIcon from '$lib/components/icons/AzureIcon.svelte' import { type Trigger, type TriggerType } from '$lib/components/triggers/utils' import { Menu, Menubar, MeltButton, MenuItem, Tooltip } from '$lib/components/meltComponents' @@ -69,6 +76,7 @@ email: { icon: Mail, countKey: 'email_count' }, nats: { icon: NatsIcon, countKey: 'nats_count', disabled: !$enterpriseLicense }, mqtt: { icon: MqttIcon, countKey: 'mqtt_count', disabled: !$enterpriseLicense }, + amqp: { icon: AmqpIcon, countKey: 'amqp_count' }, sqs: { icon: AwsIcon, countKey: 'sqs_count', disabled: !$enterpriseLicense }, gcp: { icon: GoogleCloudIcon, countKey: 'gcp_count', disabled: !$enterpriseLicense }, azure: { icon: AzureIcon, countKey: 'azure_count', disabled: !$enterpriseLicense }, @@ -103,6 +111,7 @@ 'default_email', 'nats', 'mqtt', + 'amqp', 'sqs', 'gcp', 'azure', diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index 9edc3c66bc..6723a62aa5 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -1,46 +1,58 @@ -{#if hasFilters} +{#if narrowed}
-
-
No items found
-
Try changing your search or filters
+
+
No items match the current filters
+ {#if activeFilters && activeFilters.length > 0} +
+ Active: {activeFilters.join(' · ')} +
+
Clear or widen them to see more.
+ {:else} +
Try changing your search or filters
+ {/if}
{:else} diff --git a/frontend/src/lib/components/home/TreeView.svelte b/frontend/src/lib/components/home/TreeView.svelte index aa8a3d2b14..e437ccddf1 100644 --- a/frontend/src/lib/components/home/TreeView.svelte +++ b/frontend/src/lib/components/home/TreeView.svelte @@ -1,5 +1,6 @@ -{#if isFolder(item)} +{#if isFolder(item) || isUser(item)}
(opened = !opened)} + onclick={toggleOwner} class="px-4 py-2 border-b w-full flex flex-row items-center justify-between cursor-pointer" >
0 ? `padding-left: ${depth * 16}px;` : ''} >
- {#if depth === 0} + {#if isUser(item)} + + {:else if depth === 0} {:else} @@ -69,11 +162,19 @@
- {#if depth === 0}f/{/if}{item.folderName} + + {#if isUser(item)}u/{item.username}{:else}{#if depth === 0}f/{/if}{item.folderName}{/if} +
- ({pluralize(item.items.length, ' item')}) + {#if isLazyOwner && !ownerState?.loaded} + +   + {:else if isLazyOwner && ownerState?.hasMore} + ({item.items.length}+ items) + {:else} + ({pluralize(item.items.length, ' item')}) + {/if}
@@ -100,7 +201,7 @@ Pipeline {/if} - {#each item.items.slice(0, showMax) as subItem, index ((subItem['path'] ? subItem['type'] + '__' + subItem['path'] + '__' + index : undefined) ?? 'folder__' + subItem['folderName'] + '__' + index)} + {#each item.items.slice(0, effectiveMax) as subItem, index ((subItem['path'] ? subItem['type'] + '__' + subItem['path'] + '__' + index : undefined) ?? 'folder__' + subItem['folderName'] + '__' + index)} {/each} - {#if showMax < item.items.length} + {#if effectiveMax < item.items.length}
{ - if (isFolder(item)) { - showMax += Math.min(30, item.items.length - showMax) - showMax = showMax - } + showMax += Math.min(30, item.items.length - showMax) }} > Show more ({showMax}/{item.items.length})
{/if} -
- {/if} -
-{:else if isUser(item)} -
- - -
(opened = !opened)} - class="px-4 py-2 border-b w-full flex flex-row items-center justify-between cursor-pointer" - > -
0 ? `padding-left: ${depth * 16}px;` : ''} - > -
- -
- -
- u/{item.username} -
({pluralize(item.items.length, ' item')})
-
-
-
- {#if opened} - - {:else} - - {/if} -
-
- {#if opened || isSearching} -
- {#each item.items.slice(0, showMax) as subItem, index ((subItem['path'] ? subItem['type'] + '__' + subItem['path'] + '__' + index : undefined) ?? 'folder__' + subItem['folderName'] + '__' + index)} - - {/each} - {#if showMax < item.items.length} - - -
{ - if (isUser(item)) { - showMax += Math.min(30, item.items.length - showMax) - } - }} - > - Show more ({showMax}/{item.items.length}) -
+ {#if ownerKey != undefined} + {#if ownerState?.loading && item.items.length === 0} + +
Loading…
+ {:else if !ownerState?.loading && ownerState?.hasMore && effectiveMax >= item.items.length} + + + +
ownerKey != undefined && onExpandOwner?.(ownerKey, true)} + > + Load more in {ownerKey} ({ownerState?.count ?? item.items.length} loaded) +
+ {/if} {/if}
{/if} diff --git a/frontend/src/lib/components/home/TreeViewRoot.svelte b/frontend/src/lib/components/home/TreeViewRoot.svelte index ef5cf25d58..94869b44d9 100644 --- a/frontend/src/lib/components/home/TreeViewRoot.svelte +++ b/frontend/src/lib/components/home/TreeViewRoot.svelte @@ -10,6 +10,26 @@ items: ItemType[] | undefined isSearching?: boolean pipelineFolders?: Set + sortCompare?: (a: ItemType, b: ItemType) => number + // Order of the top-level folder/user nodes: Z-A when the active sort is + // name-descending (like a file explorer), alphabetical otherwise. + groupDesc?: boolean + // The server has further pages beyond the loaded items; `onLoadMore` fetches + // the next one (grouping only reorders what's already loaded). + hasMoreServer?: boolean + onLoadMore?: () => void + // Lazy per-owner loading: every folder and every user shows as a top-level node + // regardless of the loaded window; expanding one loads its items on demand, + // paginated within it. `ownerLoad` keys are full path prefixes (`f/` / + // `u/`). + allFolders?: string[] + allUsers?: string[] + ownerLoad?: Record< + string, + { cursor?: string; hasMore: boolean; loading: boolean; loaded: boolean; count: number } + > + onExpandOwner?: (owner: string, more?: boolean) => void + onCollapseOwner?: (owner: string) => void } let { @@ -18,7 +38,16 @@ nbDisplayed = $bindable(), items, isSearching = false, - pipelineFolders + pipelineFolders, + sortCompare, + groupDesc = false, + hasMoreServer = false, + onLoadMore, + allFolders = [], + allUsers = [], + ownerLoad, + onExpandOwner, + onCollapseOwner }: Props = $props() let groupedItems: ReturnType | 'loading' = $state('loading') @@ -26,8 +55,15 @@ items pipelineFolders isSearching + sortCompare + groupDesc + allFolders + allUsers untrack(() => { - const grouped = groupItems(items) + // While searching, `items` is already relevance-ranked and the sort + // selector is disabled, so keep that order: a no-op leaf comparator + // preserves insertion order within each group (Array.sort is stable). + const grouped = groupItems(items, isSearching ? () => 0 : sortCompare, groupDesc) // Ensure every pipeline folder is present at the top level so its // "Pipeline" entry shows even when it has no listed items — a bundle-phase // pipeline (only a draft so far) or a folder whose only scripts are @@ -36,26 +72,49 @@ // hides them on `filter !== ''`), so injecting them would surface unrelated // folders in the results. if (!isSearching) { - const present = new Set( + // Inject a top-level node for every pipeline folder (so its Pipeline entry + // shows even with no listed items), every workspace folder, and every user + // — so an owner whose items sit outside the loaded window still appears; + // expanding one loads its items on demand (see onExpandOwner). Injecting + // users too is what stops a user node from vanishing under a name sort whose + // first page is all folder rows. + const presentFolders = new Set( grouped .filter((g) => 'folderName' in g) .map((g) => (g as { folderName: string }).folderName) ) - // Insert each missing pipeline folder among the existing folders in name - // order — `groupItems` already sorts user groups first then folders - // alphabetically, so inserting before the first greater-named folder - // keeps that ordering (rather than prepending out of order). - for (const folderName of [...(pipelineFolders ?? [])] - .filter((f) => !present.has(f)) - .sort()) { - const item = { folderName, items: [] } - const idx = grouped.findIndex( - (g) => - 'folderName' in g && - (g as { folderName: string }).folderName.localeCompare(folderName) > 0 + const missingFolders: { folderName: string; items: [] }[] = [] + for (const folderName of [...(pipelineFolders ?? []), ...allFolders]) { + if (presentFolders.has(folderName)) continue + presentFolders.add(folderName) + missingFolders.push({ folderName, items: [] }) + } + const presentUsers = new Set( + grouped.filter((g) => 'username' in g).map((g) => (g as { username: string }).username) + ) + const missingUsers: { username: string; items: [] }[] = [] + for (const username of allUsers) { + if (presentUsers.has(username)) continue + presentUsers.add(username) + missingUsers.push({ username, items: [] }) + } + if (missingFolders.length || missingUsers.length) { + // `groupItems` returns user groups first, then folders alphabetically. + // Append the missing nodes and sort each section once (O(n log n)) rather + // than splicing each in with findIndex (O(n²) — at 10k owners that was + // ~50M comparisons on every page merge). + const dir = groupDesc ? -1 : 1 + const users = grouped.filter((g) => 'username' in g) as { username: string }[] + const folders = grouped.filter((g) => 'folderName' in g) as { folderName: string }[] + users.push(...missingUsers) + folders.push(...missingFolders) + users.sort((a, b) => dir * a.username.localeCompare(b.username)) + folders.sort((a, b) => dir * a.folderName.localeCompare(b.folderName)) + grouped.length = 0 + grouped.push( + ...(users as unknown as typeof grouped), + ...(folders as unknown as typeof grouped) ) - if (idx < 0) grouped.push(item) - else grouped.splice(idx, 0, item) } } groupedItems = grouped @@ -74,13 +133,17 @@
{:else}
- {#each groupedItems.slice(0, nbDisplayed) as item ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} + {#each groupedItems.slice(0, nbDisplayed) as item, rootIndex ('folderName' in item ? `f__${item.folderName}` : 'username' in item ? `u__${item.username}` : `i__${item.type}__${item.path}`)} {#if item} - {#if groupedItems.length > 15 && nbDisplayed < groupedItems.length} + {#if nbDisplayed < groupedItems.length || hasMoreServer} {nbDisplayed} root nodes out of {groupedItems.length} + >{Math.min(nbDisplayed, groupedItems.length)} root nodes{hasMoreServer + ? '' + : ` out of ${groupedItems.length}`} { + if (nbDisplayed < groupedItems.length) nbDisplayed += 30 + else onLoadMore?.() + }}>load 30 more {/if} diff --git a/frontend/src/lib/components/home/treeViewUtils.ts b/frontend/src/lib/components/home/treeViewUtils.ts index 1bbe013e46..e1d769fd56 100644 --- a/frontend/src/lib/components/home/treeViewUtils.ts +++ b/frontend/src/lib/components/home/treeViewUtils.ts @@ -5,6 +5,9 @@ type TableItem = T & { type?: U time?: number starred?: boolean + // Server fetch ordinal (see ItemsList) — the tree sorts leaves by it to preserve + // the endpoint's order rather than re-deriving it. + ord?: number } type TableScript = TableItem @@ -48,7 +51,23 @@ function insertItemInFolder( }) } -export function groupItems(items: ItemType[] | undefined): (ItemType | FolderItem | UserItem)[] { +// Default leaf ordering when the caller doesn't impose one: starred first, then +// most recently modified. Folders/users always sort alphabetically regardless. +const defaultLeafCompare = (a: ItemType, b: ItemType): number => { + if (a.starred && !b.starred) return -1 + if (!a.starred && b.starred) return 1 + return getModifiedAt(b) - getModifiedAt(a) +} + +export function groupItems( + items: ItemType[] | undefined, + leafCompare: (a: ItemType, b: ItemType) => number = defaultLeafCompare, + // Folders/users have only a name, so the sort key is always name; `groupDesc` + // flips its direction (Z-A) to follow a name-descending sort, like a file explorer + // reordering folders when you reverse the name sort. Time sorts pass false (no + // folder timestamp to order by, so folders stay alphabetical). + groupDesc: boolean = false +): (ItemType | FolderItem | UserItem)[] { if (!items) { return [] } @@ -78,27 +97,37 @@ export function groupItems(items: ItemType[] | undefined): (ItemType | FolderIte } }) + const dir = groupDesc ? -1 : 1 root.sort((a, b) => { + // Users always group before folders regardless of direction; only the name + // comparison within each kind follows `groupDesc`. if ('username' in a && 'folderName' in b) { return -1 } if ('folderName' in a && 'username' in b) { return 1 } - return (a['username'] ?? a['folderName'] ?? '').localeCompare(b['username'] ?? b['folderName']) + return ( + dir * (a['username'] ?? a['folderName'] ?? '').localeCompare(b['username'] ?? b['folderName']) + ) }) - sortGroup(root) + sortGroup(root, leafCompare, dir) return root } -function sortGroup(group: (ItemType | FolderItem | UserItem)[]) { +function sortGroup( + group: (ItemType | FolderItem | UserItem)[], + leafCompare: (a: ItemType, b: ItemType) => number, + dir: number = 1 +) { group.forEach((item) => { if ('items' in item) { item.items.sort((a, b) => { + // Nested subfolders sort before leaves and follow the group direction. if ('folderName' in a && 'folderName' in b) { - return a.folderName.localeCompare(b.folderName) + return dir * a.folderName.localeCompare(b.folderName) } if ('folderName' in a) { return -1 @@ -107,14 +136,12 @@ function sortGroup(group: (ItemType | FolderItem | UserItem)[]) { return 1 } if (isItemType(a) && isItemType(b)) { - if (a.starred && !b.starred) return -1 - if (!a.starred && b.starred) return 1 - return getModifiedAt(b) - getModifiedAt(a) + return leafCompare(a, b) } return 0 }) - sortGroup(item.items) + sortGroup(item.items, leafCompare, dir) } }) } diff --git a/frontend/src/lib/components/icons/AmqpIcon.svelte b/frontend/src/lib/components/icons/AmqpIcon.svelte new file mode 100644 index 0000000000..41987f704f --- /dev/null +++ b/frontend/src/lib/components/icons/AmqpIcon.svelte @@ -0,0 +1,22 @@ + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index b39df4310e..7774bb50f0 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -104,6 +104,7 @@ import XeroIcon from './XeroIcon.svelte' import KafkaIcon from './KafkaIcon.svelte' import NatsIcon from './NatsIcon.svelte' import MqttIcon from './MqttIcon.svelte' +import AmqpIcon from './AmqpIcon.svelte' import ApifyIcon from './ApifyIcon.svelte' import McpIcon from './McpIcon.svelte' import SageIcon from './SageIcon.svelte' @@ -333,6 +334,7 @@ export const APP_TO_ICON_COMPONENT = { kafka: KafkaIcon, nats: NatsIcon, mqtt: MqttIcon, + amqp: AmqpIcon, apify: ApifyIcon, mcp: McpIcon, zoho: ZohoIcon, @@ -550,6 +552,7 @@ export { KafkaIcon, NatsIcon, MqttIcon, + AmqpIcon, ApifyIcon, McpIcon, ZohoIcon, diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 0c8ba77beb..3b892049c6 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -426,6 +426,30 @@ export const settings: Record = { cloudonly: false, ee_only: 'Workspace fairness is an Enterprise feature.', hideInQuickSetup: true + }, + { + label: 'Max jobs queued per concurrency key', + description: + 'Rejects new jobs once this many are already queued behind one concurrency key. Jobs sharing a key run at most concurrent limit at a time regardless of spare worker capacity, so a caller pushing faster than the key drains grows a backlog no capacity can absorb. Scoped per key, so a runaway producer cannot block the rest of the workspace. Set 0 to disable. Default 10000.', + key: 'concurrency_key_max_queued_jobs', + fieldType: 'number', + placeholder: '10000', + storage: 'setting', + cloudonly: true, + ee_only: '', + hideInQuickSetup: true + }, + { + label: 'Max jobs queued per workspace', + description: + 'Rejects new jobs once a workspace has this many queued in total, across every concurrency key and script. Guards against a single workspace flooding the queue generally, including from parallel for-loops. Applies even to premium workspaces. Jobs already queued still drain; only new pushes past the ceiling are rejected. Set 0 to disable. Default 20000.', + key: 'workspace_max_queued_jobs', + fieldType: 'number', + placeholder: '20000', + storage: 'setting', + cloudonly: true, + ee_only: '', + hideInQuickSetup: true } ], 'Object Storage': [ diff --git a/frontend/src/lib/components/markdownProse.ts b/frontend/src/lib/components/markdownProse.ts new file mode 100644 index 0000000000..bc943aa42b --- /dev/null +++ b/frontend/src/lib/components/markdownProse.ts @@ -0,0 +1,36 @@ +/** + * Shared prose (Tailwind typography) stacks for markdown renders, one source of + * truth so surfaces don't each roll their own and drift apart. Compose at the + * call site with layout-only classes (padding, width, bg); anything typographic + * belongs here. + * + * - 'xs': micro scale for dense secondary panes (chat reasoning blocks) + * - 'sm': compact chat-bubble scale (assistant messages, flow/app chat, settings) + * - 'doc': same rhythm and body size as 'sm', with a taller heading ramp + * (lg/base/sm) and semibold headings for document-like surfaces (artifacts) + */ + +// Kept as literal template parts: Tailwind's scanner reads class names verbatim +// from this file, so every token must appear as plain text. +// content-none strips the typography plugin's decorative backticks around +// inline code (code::before/::after), which read as literal ` characters. +// [&>:first-child]:mt-0 re-applies the plugin's first-block reset, which our +// explicit prose-headings:mt-* would otherwise override (note: the composed +// variant prose-headings:first: attaches :first-child to the wrapper — wrong). +const base = + 'prose dark:prose-invert max-w-full break-words [&>:first-child]:mt-0 prose-a:break-words prose-code:break-words prose-code:before:content-none prose-code:after:content-none prose-code:bg-surface-secondary/50 prose-code:rounded prose-code:px-1 prose-code:py-0.5 prose-code:font-normal prose-table:block prose-table:max-w-full prose-table:overflow-x-auto' + +// One vertical rhythm for sm/doc; heading margins stay per-preset (fixed, not +// the plugin's em-based ones) so 'doc' can breathe more between sections. +const rhythm = 'prose-sm leading-snug prose-ul:!pl-6' + +const bodyXs = + 'text-primary prose-p:text-primary prose-li:text-primary prose-p:text-xs prose-li:text-xs prose-code:text-xs prose-pre:text-xs prose-table:text-xs' + +export const markdownProse = { + xs: `${base} prose-sm leading-snug prose-ul:!pl-5 prose-p:text-2xs prose-li:text-2xs prose-code:text-2xs prose-pre:text-2xs prose-headings:font-medium prose-headings:text-secondary prose-headings:mt-2 prose-headings:mb-1 prose-h1:text-2xs prose-h2:text-2xs prose-h3:text-2xs prose-h4:text-2xs prose-h5:text-2xs prose-h6:text-2xs prose-strong:text-secondary`, + sm: `${base} ${rhythm} ${bodyXs} prose-headings:mt-3 prose-headings:mb-1 prose-headings:font-medium prose-headings:text-emphasis prose-h1:text-sm prose-h2:text-xs prose-h3:text-xs prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs`, + doc: `${base} ${rhythm} ${bodyXs} prose-headings:mt-8 prose-headings:mb-2 prose-headings:font-semibold prose-headings:text-emphasis prose-h1:text-lg prose-h2:text-base prose-h3:text-sm prose-h4:text-xs prose-h5:text-xs prose-h6:text-xs prose-pre:bg-transparent prose-pre:p-0` +} as const + +export type MarkdownProseSize = keyof typeof markdownProse diff --git a/frontend/src/lib/components/mcp/McpScopeSelector.svelte b/frontend/src/lib/components/mcp/McpScopeSelector.svelte index cf95617790..3ff1ab3d95 100644 --- a/frontend/src/lib/components/mcp/McpScopeSelector.svelte +++ b/frontend/src/lib/components/mcp/McpScopeSelector.svelte @@ -1,5 +1,5 @@ + + + drawer?.closeDrawer()}> +
+

+ Declared with // measure and // dimension on the script that materializes + the table. Inserting writes plain SQL you can edit. +

+ + {#if loading} + Loading… + {:else if loaded && tables.length === 0} +
+ No metrics declared yet + + Add // measure revenue = sum(amount) where not is_refund next to a script's + // materialize annotation, then deploy it. + +
+ {/if} + + {#if tables.length > 0} +
+
+ {tables.length === 1 ? 'Table' : `Tables with metrics (${tables.length})`} +
+ {#if tables.length > TABLE_FILTER_MIN} +
+ +
+ {/if} + +
+ {#each shownTables as t (t)} + + {/each} +
+ {#if shownTables.length === 0} + No table matches “{tableFilter}” + {/if} +
+ {/if} + + {#if measures.length + dimensions.length > METRIC_FILTER_MIN} + + {/if} + + {#if shownMeasures.length > 0} +
+
Measures
+
+ {#each shownMeasures as m (idOf(m))} +
+ toggle(selectedMeasures, idOf(m))} + /> + + = {definition(m)} + +
+ {/each} +
+
+ {/if} + + {#if shownDimensions.length > 0} +
+
Group by
+
+ {#each shownDimensions as d (idOf(d))} +
+ toggle(selectedDims, idOf(d))} + /> + + = {d.expr} + +
+ {/each} +
+
+ {/if} + + {#if metricFilter.trim() && shownMeasures.length === 0 && shownDimensions.length === 0 && measures.length + dimensions.length > 0} + No metric matches “{metricFilter}” + {/if} + + {#if sql} +
+
+ SQL +
+ + +
+
+
{sql}
+ {#if insertAdapts} +
+ Inserting reuses this script's existing {existingAlias} + attachment and leaves out the ATTACH. +
+ {/if} +
+ {/if} + + {#if lake} +
+
Try it
+
+ (preview = { sql: ranCode, data: d })} + /> +
+ {#if preview && preview.sql === replSql} + +
+ +
+ {/if} +
+ {/if} +
+ + {#snippet actions()} + + {/snippet} +
+
diff --git a/frontend/src/lib/components/metrics/metricSql.test.ts b/frontend/src/lib/components/metrics/metricSql.test.ts new file mode 100644 index 0000000000..0d19344b21 --- /dev/null +++ b/frontend/src/lib/components/metrics/metricSql.test.ts @@ -0,0 +1,181 @@ +import { describe, expect, it } from 'vitest' +import { attachAliasFor, attachAliasTaken, composeMetricSql, pickAttachAlias } from './metricSql' +import type { DataMetric } from '$lib/gen' + +const revenue: DataMetric = { + script_path: 'f/metrics/orders', + table_path: 'sales/main.orders', + kind: 'measure', + name: 'revenue', + expr: 'sum(amount)', + filter: 'not is_refund' +} +const region: DataMetric = { + script_path: 'f/metrics/orders', + table_path: 'sales/main.orders', + kind: 'dimension', + name: 'region', + expr: 'region' +} + +describe('attachAliasFor', () => { + it('ignores ATTACH-like text inside a string or dollar-quoted literal', () => { + // The `$$…$$` body is data, not a statement. + expect( + attachAliasFor("SELECT $$ATTACH 'ducklake://sales' AS ghost;$$;", 'sales') + ).toBeUndefined() + expect(attachAliasTaken("SELECT $$ATTACH 'x.db' AS dl;$$;", 'dl')).toBe(false) + // A real ATTACH alongside literal noise is still found. + const code = "SELECT '--not an attach';\nATTACH 'ducklake://sales' AS dl;" + expect(attachAliasFor(code, 'sales')).toBe('dl') + }) + + it('ignores an ATTACH that is commented out, in any of the comment styles', () => { + for (const comment of ['--', '//']) { + const code = `${comment} ATTACH 'ducklake://sales' AS old;\nATTACH 'ducklake://sales' AS dl;` + expect(attachAliasFor(code, 'sales')).toBe('dl') + } + expect(attachAliasFor("/* ATTACH 'ducklake://sales' AS old; */", 'sales')).toBeUndefined() + }) + + it('resolves the default-lake shorthand, which refers to the lake named main', () => { + expect(attachAliasFor("ATTACH 'ducklake' AS dl;", 'main')).toBe('dl') + expect(attachAliasFor("ATTACH 'ducklake://main' AS dl;", 'main')).toBe('dl') + // The shorthand must not be read as an attachment of some other lake. + expect(attachAliasFor("ATTACH 'ducklake' AS dl;", 'sales')).toBeUndefined() + }) + + it('matches the lake name case-sensitively, unlike the SQL keywords', () => { + // `sales` and `Sales` are distinct DuckLake configs. + expect(attachAliasFor("attach 'ducklake://sales' as dl;", 'sales')).toBe('dl') + expect(attachAliasFor("ATTACH 'ducklake://sales' AS dl;", 'Sales')).toBeUndefined() + expect(attachAliasFor("ATTACH 'ducklake://Sales' AS dl;", 'sales')).toBeUndefined() + }) + + it('does not treat mid-line // (integer division) as a comment', () => { + // `5 // 2` is division; the following ATTACH must still be found. + const code = "SELECT 5 // 2;\nATTACH 'ducklake://sales' AS dl;" + expect(attachAliasFor(code, 'sales')).toBe('dl') + // A leading `// materialize` annotation is still a comment. + expect(attachAliasFor("// materialize x\nATTACH 'ducklake://sales' AS dl;", 'sales')).toBe('dl') + }) + + it('reads a doubled-quote alias as one identifier', () => { + expect(attachAliasFor(`ATTACH 'ducklake://sales' AS "a""b";`, 'sales')).toBe('a"b') + }) + + it('does not let a backslash-ending literal hide a following ATTACH', () => { + // `'C:\'` is a complete SQL literal (quotes escape by doubling, not `\`); the + // ATTACH after it must still be found, and its alias seen as taken. + const code = "SELECT 'C:\\';\nATTACH 'ducklake://sales' AS dl;" + expect(attachAliasFor(code, 'sales')).toBe('dl') + expect(attachAliasTaken(code, 'dl')).toBe(true) + // A doubled quote embeds a literal quote without ending the string. + expect(attachAliasFor("SELECT 'a''b';\nATTACH 'ducklake://sales' AS dl;", 'sales')).toBe('dl') + }) + + it('recognizes the optional ATTACH DATABASE keyword', () => { + expect(attachAliasFor("ATTACH DATABASE 'ducklake://sales' AS dl;", 'sales')).toBe('dl') + expect(attachAliasTaken("ATTACH DATABASE 'x.db' AS dl;", 'dl')).toBe(true) + }) + + it('reads a quoted alias whole, including one containing a space', () => { + expect(attachAliasFor(`ATTACH 'ducklake://sales' AS "my dl";`, 'sales')).toBe('my dl') + expect(attachAliasFor(`ATTACH 'ducklake://sales' AS dl;`, 'sales')).toBe('dl') + }) +}) + +describe('pickAttachAlias', () => { + it('skips aliases already attached, whatever they are attached to', () => { + expect(pickAttachAlias('SELECT 1;', 'sales')).toBe('dl') + // `dl` taken by another lake: fall back to the lake's own name. + expect(pickAttachAlias("ATTACH 'ducklake://other' AS dl;", 'sales')).toBe('sales') + // Both taken, and the second by a non-DuckLake attachment, which occupies + // the alias just as effectively. + expect( + pickAttachAlias("ATTACH 'ducklake://other' AS dl;\nATTACH 'foo.db' AS sales;", 'sales') + ).toBe('dl2') + }) + + it('does not treat a longer alias as a match for a shorter one', () => { + expect(attachAliasTaken("ATTACH 'x.db' AS dl2;", 'dl')).toBe(false) + }) +}) + +describe('attachAliasFor with tricky sources', () => { + it('still finds a real attachment when a literal contains comment markers', () => { + const code = "SELECT '--not a comment';\nATTACH 'ducklake://sales' AS dl;" + expect(attachAliasFor(code, 'sales')).toBe('dl') + }) +}) + +describe('composeMetricSql', () => { + it('qualifies the table with the alias the script already attached the lake under', () => { + // DuckDB names the catalog after the ATTACH alias, so qualifying with the + // lake name would not resolve against a script that attached it as `dl`. + const code = "ATTACH 'ducklake://sales' AS dl;\nSELECT 1;" + const sql = composeMetricSql({ + tablePath: 'sales/main.orders', + measures: [revenue], + dimensions: [], + existingAlias: attachAliasFor(code, 'sales') + }) + expect(sql).toContain('FROM "dl"."main"."orders"') + expect(sql).not.toContain('ATTACH') + }) + + it('attaches the lake under the conventional dl alias when the script has none', () => { + const sql = composeMetricSql({ + tablePath: 'sales/main.orders', + measures: [revenue], + dimensions: [], + existingAlias: attachAliasFor('SELECT 1;', 'sales') + }) + expect(sql).toContain(`ATTACH 'ducklake://sales' AS "dl";`) + expect(sql).toContain('FROM "dl"."main"."orders"') + }) + + it('falls back to the lake name when the script already binds dl elsewhere', () => { + const code = "ATTACH 'ducklake://other' AS dl;" + const sql = composeMetricSql({ + tablePath: 'sales/main.orders', + measures: [revenue], + dimensions: [], + existingAlias: attachAliasFor(code, 'sales'), + attachAs: attachAliasTaken(code, 'dl') ? 'sales' : 'dl' + }) + expect(sql).toContain(`ATTACH 'ducklake://sales' AS "sales";`) + expect(sql).toContain('FROM "sales"."main"."orders"') + }) + + it('quotes every generated identifier, so reserved words are safe', () => { + const reservedMeasure: DataMetric = { + ...revenue, + name: 'select', + expr: 'count(*)', + filter: undefined + } + const reservedDim: DataMetric = { ...region, name: 'group', expr: 'region' } + const sql = composeMetricSql({ + // `order` is a reserved word, and so is the `select`/`group` naming below. + tablePath: 'sales/main.order', + measures: [reservedMeasure], + dimensions: [reservedDim], + existingAlias: 'dl' + }) + expect(sql).toContain('AS "select"') + expect(sql).toContain('AS "group"') + expect(sql).toContain('FROM "dl"."main"."order"') + }) + + it('renders a measure predicate as FILTER so measures can share one GROUP BY', () => { + const sql = composeMetricSql({ + tablePath: 'sales/main.orders', + measures: [revenue], + dimensions: [region], + existingAlias: 'dl' + }) + expect(sql).toContain('sum(amount) FILTER (WHERE not is_refund) AS "revenue"') + expect(sql).toContain('GROUP BY 1') + }) +}) diff --git a/frontend/src/lib/components/metrics/metricSql.ts b/frontend/src/lib/components/metrics/metricSql.ts new file mode 100644 index 0000000000..7cce1c2075 --- /dev/null +++ b/frontend/src/lib/components/metrics/metricSql.ts @@ -0,0 +1,182 @@ +import type { DataMetric } from '$lib/gen' +import { splitSqlStatements, stripSqlComments } from '../sqlDdl' + +function quoteIdent(name: string): string { + return `"${name.replaceAll('"', '""')}"` +} + +/** + * All generated identifiers are quoted. Quoting conditionally would mean tracking + * DuckDB's reserved-word set, which shifts between versions: a name like `select`, + * `order` or `lambda` is rejected bare, and a list that misses one silently emits + * invalid SQL. Quoting always is version-proof and needs no maintenance. + */ +function ident(name: string): string { + return quoteIdent(name) +} + +/** + * Alias used when the snippet attaches the lake itself. `dl` is the convention + * across the codebase's DuckLake scripts; the lake's own name is only a fallback + * for when the script already binds `dl` to a different lake. + */ +export const DEFAULT_LAKE_ALIAS = 'dl' + +function escapeRegex(s: string): string { + return s.replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +type TableParts = { lake: string; schema: string; table: string } + +/** Splits `/
` or `/.
`, defaulting the schema. */ +export function splitTablePath(tablePath: string): TableParts | undefined { + const path = tablePath.startsWith('ducklake://') + ? tablePath.slice('ducklake://'.length) + : tablePath + const slash = path.indexOf('/') + if (slash < 0) return undefined + const lake = path.slice(0, slash) + const rest = path.slice(slash + 1) + const dot = rest.indexOf('.') + const schema = dot < 0 ? 'main' : rest.slice(0, dot) + const table = dot < 0 ? rest : rest.slice(dot + 1) + if (!lake || !table) return undefined + return { lake, schema, table } +} + +/** + * The alias `code` attaches `lake` under, if it attaches it at all. + * + * DuckDB names the catalog after the ATTACH alias, not after the lake, and the + * prevailing convention is an arbitrary short alias (`AS dl`). Qualifying a table + * with the lake name would therefore fail against most real scripts. + * + * `ATTACH 'ducklake'` with no `://` part refers to the lake named `main`, so that + * shorthand has to resolve here too or a `main` table gets a redundant second + * attachment under a different alias. + */ +// SQL statements, comments stripped, split on real (non-literal) `;`. Matching +// ATTACH per statement (anchored at its start) ignores attachment-like text +// buried in a string or dollar-quoted literal, e.g. `SELECT $$ATTACH … AS x$$`. +// `false` selects standard-SQL string quoting: this is DuckDB, where `\` is data +// (so a `'C:\'` literal does not swallow the following ATTACH) and quotes double. +function statements(code: string): string[] { + return splitSqlStatements(stripSqlComments(code, true, false), false).map((s) => s.trim()) +} + +export function attachAliasFor(code: string, lake: string): string | undefined { + // SQL keywords match case-insensitively; the lake name must not. DuckLake + // config keys are case-sensitive, so `sales` and `Sales` are different lakes, + // and folding them would reuse an alias bound to the wrong one. So the URI is + // captured and compared exactly. A `ducklake` with no `://` refers to `main`. + // A quoted alias may contain doubled quotes (`"a""b"` = the identifier `a"b`); + // capture the whole quoted run and unescape, and require an unquoted alias to + // end on a token boundary. + const re = + /^ATTACH\s+(?:DATABASE\s+)?(?:IF\s+NOT\s+EXISTS\s+)?'ducklake(:\/\/[^']*)?'\s+AS\s+(?:"((?:[^"]|"")*)"|([A-Za-z_][\w$]*))(?![\w$])/i + for (const stmt of statements(code)) { + const m = re.exec(stmt) + if (m) { + const attachedLake = m[1] ? m[1].slice('://'.length) : 'main' + if (attachedLake === lake) { + return m[2] !== undefined ? m[2].replaceAll('""', '"') : m[3] + } + } + } + return undefined +} + +/** + * Whether `code` already attaches anything under `alias`. Deliberately not + * limited to DuckLake: a SQLite or Postgres attachment occupies the alias just + * as effectively, and re-using it would fail on the duplicate name. + */ +export function attachAliasTaken(code: string, alias: string): boolean { + const re = new RegExp( + `^ATTACH\\s+(?:DATABASE\\s+)?(?:IF\\s+NOT\\s+EXISTS\\s+)?'[^']*'\\s+AS\\s+"?${escapeRegex(alias)}"?(?![\\w$])`, + 'i' + ) + return statements(code).some((stmt) => re.test(stmt)) +} + +/** + * First alias not already attached in `code`: the conventional `dl`, then the + * lake's own name, then numbered variants. Checking each matters because falling + * back blindly can land on a name that is itself taken. + */ +export function pickAttachAlias(code: string, lake: string): string { + if (!attachAliasTaken(code, DEFAULT_LAKE_ALIAS)) return DEFAULT_LAKE_ALIAS + if (!attachAliasTaken(code, lake)) return lake + // Unbounded on purpose, and it terminates: a finite script can only occupy + // finitely many aliases, so some `dlN` is always free. Returning a taken one + // instead would emit an ATTACH that fails on the duplicate name. + let n = 2 + while (attachAliasTaken(code, `${DEFAULT_LAKE_ALIAS}${n}`)) n += 1 + return `${DEFAULT_LAKE_ALIAS}${n}` +} + +/** + * Whether `code` looks like it references this table. Used only to preselect a + * table in the picker, so a false positive costs nothing. + */ +export function codeMentionsTable(code: string, tablePath: string): boolean { + const parts = splitTablePath(tablePath) + if (!parts) return false + return new RegExp(`\\b${escapeRegex(parts.table)}\\b`, 'i').test(code) +} + +/** + * A measure renders as its declared aggregate plus, when it carries a predicate, + * a trailing `FILTER (WHERE …)`. FILTER rather than a shared `WHERE` is what lets + * measures with different predicates sit under one GROUP BY. + */ +function measureExpr(m: DataMetric): string { + return m.filter ? `${m.expr} FILTER (WHERE ${m.filter})` : m.expr +} + +/** + * Composes a plain SELECT from a metric selection. The result is ordinary + * editable SQL: nothing re-reads or rewrites it later, so a user is free to + * change it after inserting. + */ +export function composeMetricSql(opts: { + tablePath: string + measures: DataMetric[] + dimensions: DataMetric[] + /** + * The alias the script already attaches this lake under. When absent the + * snippet attaches the lake itself under `attachAs`. + */ + existingAlias?: string + /** Alias to attach under when the script has none. Defaults to `dl`. */ + attachAs?: string +}): string { + const { tablePath, measures, dimensions, existingAlias, attachAs } = opts + const parts = splitTablePath(tablePath) + if (!parts) return '' + // Qualify with the alias in force, so the snippet resolves against the + // catalog the script actually has attached. + const catalog = existingAlias ?? attachAs ?? DEFAULT_LAKE_ALIAS + const tableRef = `${ident(catalog)}.${ident(parts.schema)}.${ident(parts.table)}` + + const selected = [ + ...dimensions.map((d) => ` ${d.expr} AS ${ident(d.name)}`), + ...measures.map((m) => ` ${measureExpr(m)} AS ${ident(m.name)}`) + ] + + const lines: string[] = [] + if (!existingAlias) { + // Escape the lake as a SQL string literal. The deploy path already rejects + // unsafe names, but the composed query is copied and run standalone, so it + // must not trust the catalog value blindly. + const lakeLiteral = parts.lake.replaceAll("'", "''") + lines.push(`ATTACH 'ducklake://${lakeLiteral}' AS ${ident(catalog)};`, '') + } + lines.push('SELECT', selected.join(',\n'), `FROM ${tableRef}`) + if (dimensions.length > 0) { + // Group by ordinal so a dimension expression is written once. + const ordinals = dimensions.map((_, i) => i + 1).join(', ') + lines.push(`GROUP BY ${ordinals}`, `ORDER BY ${ordinals}`) + } + return lines.join('\n') +} diff --git a/frontend/src/lib/components/offboarding-utils.ts b/frontend/src/lib/components/offboarding-utils.ts index f6f41bcc68..11f266e101 100644 --- a/frontend/src/lib/components/offboarding-utils.ts +++ b/frontend/src/lib/components/offboarding-utils.ts @@ -28,6 +28,7 @@ const TRIGGER_TABLE_TO_ROUTE: Record = { kafka_trigger: 'kafka_triggers', postgres_trigger: 'postgres_triggers', mqtt_trigger: 'mqtt_triggers', + amqp_trigger: 'amqp_triggers', nats_trigger: 'nats_triggers', sqs_trigger: 'sqs_triggers', gcp_trigger: 'gcp_triggers', @@ -41,6 +42,7 @@ const TRIGGER_TABLE_TO_LABEL: Record = { kafka_trigger: 'kafka trigger', postgres_trigger: 'postgres trigger', mqtt_trigger: 'mqtt trigger', + amqp_trigger: 'amqp trigger', nats_trigger: 'nats trigger', sqs_trigger: 'sqs trigger', gcp_trigger: 'gcp trigger', diff --git a/frontend/src/lib/components/raw_apps/InlineElementPrompt.svelte b/frontend/src/lib/components/raw_apps/InlineElementPrompt.svelte new file mode 100644 index 0000000000..33a6617ace --- /dev/null +++ b/frontend/src/lib/components/raw_apps/InlineElementPrompt.svelte @@ -0,0 +1,74 @@ + + + + + diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 0be384169e..0ef8ac5a58 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -23,8 +23,11 @@ type RawAppRuntimeLogEntry, type RawAppRuntimeLogRequester, type RawAppRunSummary, - type RawAppRunsProvider + type RawAppRunsProvider, + type RawAppScreenshotRequester } from './utils' + import { runDomQueryOnHtml, type RawAppDomQuery, type RawAppDomRequester } from './rawAppDom' + import InlineElementPrompt from './InlineElementPrompt.svelte' import DarkModeObserver from '../DarkModeObserver.svelte' import RawAppSidebar from './RawAppSidebar.svelte' import type { Modules } from './RawAppModules.svelte' @@ -37,6 +40,7 @@ InspectorElementInfo } from '../copilot/chat/app/core' import { createAppSelectedContext, type AppCodeSelectionElement } from '../copilot/chat/context' + import { captureScale, MAX_IMAGE_EDGE } from '../copilot/chat/imageUtils' import { rawAppLintStore } from './lintStore' import { dbSchemas } from '$lib/stores' import { @@ -125,6 +129,21 @@ onOpenOthersDrafts?: () => void onRuntimeLogRequester?: (requester: RawAppRuntimeLogRequester | undefined) => void onRunsProvider?: (provider: RawAppRunsProvider | undefined) => void + // Session preview only: expose a live DOM query requester (search/read the + // rendered preview by CSS selector) and forward inspector element picks so + // they can be attached to the session chat as selector context chips. + onDomRequester?: (requester: RawAppDomRequester | undefined) => void + // `additive` (Shift held) adds to the selection; otherwise it replaces it. + onInspectorSelect?: (info: InspectorElementInfo, additive: boolean) => void + // Session preview only: the chat's current DOM-selector chips (source of + // truth). Pushed into the preview so it renders one highlight per selector. + selectedDomSelectors?: string[] + onInspectorDeselect?: (selector: string) => void + onInspectorClearAll?: () => void + // Session preview only: send a prompt scoped to a selected element (via the + // inline mini-composer anchored over it in the preview). + onInlinePrompt?: (selector: string, prompt: string) => void + onScreenshotRequester?: (requester: RawAppScreenshotRequester | undefined) => void // Restoring an older deployment from the history drawer. A callback prop // (not `on:restore` forwarding): forwarding a `createEventDispatcher` // event up through these runes-mode components silently drops it. @@ -166,6 +185,13 @@ onOpenOthersDrafts, onRuntimeLogRequester = undefined, onRunsProvider = undefined, + onDomRequester = undefined, + onInspectorSelect = undefined, + selectedDomSelectors = [], + onInspectorDeselect = undefined, + onInspectorClearAll = undefined, + onInlinePrompt = undefined, + onScreenshotRequester = undefined, onRestore, onSavedNewAppPath, condensedHeader = false @@ -291,6 +317,29 @@ let buildError = $state(undefined) // Latest uncaught runtime error thrown by the rendered app; cleared on next build. let runtimeError = $state(undefined) + // Set when a build ran cleanly but never mounted anything into #root — the + // entrypoint defines a component without ever mounting it. Cleared on next build. + let emptyRender = $state(false) + // The repair hint above has to match the app's framework: only React apps have + // an `index.tsx` and `createRoot`; Svelte and Vue mount from `index.ts`. Keyed + // off file extensions rather than exact template filenames, which users rename. + let mountHint = $derived.by(() => { + const paths = Object.keys(files ?? {}).map((p) => p.replace(/^\//, '')) + const entrypoint = paths.find((p) => /^index\.(tsx|jsx|ts|js)$/.test(p)) + if (paths.some((p) => p.endsWith('.svelte'))) { + return { + entrypoint: entrypoint ?? 'index.ts', + call: "mount(App, { target: document.getElementById('root')! })" + } + } + if (paths.some((p) => p.endsWith('.vue'))) { + return { entrypoint: entrypoint ?? 'index.ts', call: "createApp(App).mount('#root')" } + } + return { + entrypoint: entrypoint ?? 'index.tsx', + call: "createRoot(document.getElementById('root')!).render()" + } + }) let logsCollapsed = $state(false) let logsDiv: HTMLDivElement | undefined = $state(undefined) $effect(() => { @@ -1099,6 +1148,18 @@ return } + // The build ran without mounting the app, so the preview is blank with + // nothing to report — surfaced as a hint naming the missing mount call. + // `renderAppeared` withdraws it if a mount lands after the grace window. + if (fromPreview && e.data.type === 'emptyRender') { + emptyRender = true + return + } + if (fromPreview && e.data.type === 'renderAppeared') { + emptyRender = false + return + } + // Uncaught error/rejection from the rendered app — surfaced in the preview // overlay so a runtime crash isn't a silent blank error. if (fromPreview && e.data.type === 'runtimeError') { @@ -1112,11 +1173,24 @@ // Inspector events come exclusively from the preview iframe. if (fromPreview && e.data.type === 'inspectorSelect') { inspectorElement = e.data.element as InspectorElementInfo - inspectorEnabled = false + // Session preview: forward the pick so it can be attached to the chat as a + // selector context chip (the app-mode SelectedContext path is separate). + // App mode picks one element then exits; the session stays on to keep + // picking (the chip list, not the harness, holds the selection). Shift + // held → add to the selection; a plain click replaces it. + if (onInspectorSelect) onInspectorSelect(inspectorElement, !!e.data.additive) + else inspectorEnabled = false + return + } + if (fromPreview && e.data.type === 'inspectorDeselect') { + // User clicked × on a selected overlay in the preview — drop that chip. + if (typeof e.data.selector === 'string') onInspectorDeselect?.(e.data.selector) return } if (fromPreview && e.data.type === 'inspectorClear') { inspectorElement = undefined + // A rebuild invalidates the selection — clear all session chips. + onInspectorClearAll?.() return } @@ -1198,11 +1272,12 @@ } } - // Feed a build into the inline preview iframe. Clears any prior runtime-error - // overlay first: a fresh render supersedes the old crash, and if the new - // render throws again app-preview.html re-posts `runtimeError`. + // Feed a build into the inline preview iframe. Clears the previous run's + // overlays first: a fresh render supersedes the old crash or blank, and + // app-preview.html re-posts if the new render fails the same way. function feedPreviewIframe(build: { css: string; js: string }) { runtimeError = undefined + emptyRender = false previewIframe?.contentWindow?.postMessage( { type: 'preview', css: build.css, js: build.js }, '*' @@ -1298,6 +1373,172 @@ }) } + // Live DOM inspection for the session chat. Same-origin: the preview iframe is a + // same-origin document (see the load listener below that reads its contentWindow), + // so we read `contentDocument` directly — no postMessage, no ui_builder change. The + // element is re-read on every call, so the model always sees the current render. + const requestDomQuery: RawAppDomRequester = async (query: RawAppDomQuery) => { + const doc = previewIframe?.contentDocument + if (!doc || !previewIframeLoaded) return undefined + const selector = query.selector?.trim() + let el: Element | null + let matchCount: number + if (selector) { + let matches: NodeListOf + try { + matches = doc.querySelectorAll(selector) + } catch (e) { + return { + text: `Invalid CSS selector "${selector}": ${e instanceof Error ? e.message : String(e)}` + } + } + matchCount = matches.length + el = matches[0] ?? null + } else { + el = doc.body + matchCount = el ? 1 : 0 + } + if (!el) { + return { + text: `No element matches selector "${selector}". It may not be rendered yet, or the selector is wrong. Try a broader selector or omit it to read the whole page.` + } + } + // A + +
+ {#if !hideControls} +
+
+

+ {replayState === 'playing' ? 'Replaying: ' : ''}{recording.folder || 'Pipeline'} +

+ + + {#snippet text()} + + Recorded {new Date(recording.recorded_at).toLocaleString()} — + {(recording.total_duration_ms / 1000).toFixed(1)}s — + {Object.keys(recording.jobs).length} job(s) + + {/snippet} + +
+ {#if replayState === 'playing'} + + {:else if replayState === 'done'} +
+ + +
+ {:else} + + {/if} +
+ {/if} + + + + +
+ + +
+ (selection = s)} + {activeRunnableIds} + {runStates} + highlightActiveRun={replayState === 'playing'} + {recomputedAssetIds} + showMinimap={!stacked} + viewportFitKey={recording.folder} + /> + {#if !selection} +

+ {replayState === 'playing' + ? 'Replaying the recorded run — click a node to inspect its logs, result or data sample.' + : 'Click a script node for its logs/result, or an asset node for its recorded data sample.'} +

+ {/if} +
+
+ {#if selection} + +
+ {#if selection.kind === 'runnable'} +
+
+ {#if selectedStatus === 'running'} + + {:else if selectedStatus === 'success'} + + {:else if selectedStatus === 'failure'} + + {/if} + {selection.path} +
+ (runnableTab = e.detail)} + > + {#snippet children({ item })} + + + {/snippet} + +
+
+ + {#if runnableTab === 'code'} +
+ {#if selectedCode} + + {:else} +

+ No code was captured for this step in the recording. +

+ {/if} +
+ {:else if selectedJobId && recording.jobs[selectedJobId]} + {#if job?.args && Object.keys(job.args).length > 0} +
+

Arguments

+ +
+ {/if} + +
+

Logs

+
+ +
+
+
+

Result

+
+ {#if job !== undefined && job.type === 'CompletedJob' && job.result !== undefined} + + {:else if jobDone} +
No output available
+ {:else} +
Waiting for result…
+ {/if} +
+
+ {:else} +

This node did not run in the recorded session.

+ {/if} +
+ {:else if selection.kind === 'asset'} +
+
+ + {selectedAssetSample?.uri ?? + `${selection.asset_kind}://${selection.path}`} + {#if selectedAssetSample && !selectedAssetSample.error} + + {selectedAssetSample.rowCount != undefined + ? `${selectedAssetSample.rowCount} row${selectedAssetSample.rowCount === 1 ? '' : 's'}` + : `${selectedAssetSample.rows.length} sampled`} + · {selectedAssetSample.columns.length} column{selectedAssetSample.columns + .length === 1 + ? '' + : 's'} + + {/if} +
+ + {#if !selectedAssetSample} +

+ No data sample was captured for this asset in the recorded session. +

+ {:else if selectedAssetSample.error} +

+ Could not capture a sample of this table: {selectedAssetSample.error} +

+ {:else if selectedAssetSample.rows.length === 0} +

This table was empty when the recording was taken.

+ {:else} +
+
+ + + {#each selectedAssetSample.columns as c} + + {/each} + + + + {#each selectedAssetSample.rows as row} + + {#each selectedAssetSample.columns as c} + + {/each} + + {/each} + +
+ {c.field} + {#if c.datatype}· {c.datatype}{/if} +
+ {fmtCell((row as any)?.[c.field])} +
+
+ {/if} + + {/if} + + + {/if} + + + diff --git a/frontend/src/lib/components/recording/pipelineAssetSample.ts b/frontend/src/lib/components/recording/pipelineAssetSample.ts new file mode 100644 index 0000000000..a5b9cf6456 --- /dev/null +++ b/frontend/src/lib/components/recording/pipelineAssetSample.ts @@ -0,0 +1,77 @@ +import type { AssetKind } from '$lib/gen' +import { parseDbInputFromAssetSyntax } from '$lib/utils' +import { loadAllTablesMetaData } from '$lib/components/apps/components/display/dbtable/metadata' +import { dbTableOpsWithPreviewScripts } from '$lib/components/dbOps' +import type { PipelineAssetSample } from './types' + +// How many rows to sample per asset — a preview, not a dump. +const SAMPLE_LIMIT = 100 + +/** + * Capture a data-sample of a pipeline asset for the recorder, reusing the exact + * same query path the live asset-preview panes use (`loadAllTablesMetaData` + + * `dbTableOpsWithPreviewScripts.getRows`), so a replayed sample matches what the + * pane would have shown. Only ducklake / datatable assets are sampleable this + * way (s3object files use a different preview); other kinds return an error + * marker the player renders as "no sample". + * + * Never throws — a failed capture (missing table, unconfigured datatable) is + * returned as a `PipelineAssetSample` with `error` set so the recording still + * completes. + */ +export async function capturePipelineAssetSample( + workspace: string, + kind: AssetKind, + path: string +): Promise { + const uri = `${kind}://${path}` + const base: PipelineAssetSample = { kind, path, uri, columns: [], rows: [] } + if (kind !== 'ducklake' && kind !== 'datatable') { + return { ...base, error: `no sample for ${kind} assets` } + } + try { + const input = parseDbInputFromAssetSyntax(uri) + if (!input) return { ...base, error: 'could not parse asset uri' } + const table = 'specificTable' in input ? (input.specificTable as string | undefined) : undefined + const schema = + 'specificSchema' in input ? (input.specificSchema as string | undefined) : undefined + if (!table) return { ...base, error: 'asset uri has no table' } + + const defs = await loadAllTablesMetaData(workspace, input) + if (!defs) return { ...base, error: 'table metadata unavailable' } + + // Same table-key resolution as the ducklake/datatable preview panes: + // try `schema.table` then bare `table`, then any key ending in `.table`. + const defaultSchema = kind === 'ducklake' ? 'main' : 'public' + const colDefs = + defs[`${schema ?? defaultSchema}.${table}`] ?? + defs[table] ?? + (() => { + const key = Object.keys(defs).find((k) => k === table || k.endsWith(`.${table}`)) + return key ? defs[key] : undefined + })() + if (!colDefs) return { ...base, error: 'table does not exist yet' } + + const tableKey = schema && table ? `${schema}.${table}` : table + const ops = dbTableOpsWithPreviewScripts({ input, tableKey, colDefs, workspace }) + const rows = await ops.getRows({ + offset: 0, + limit: SAMPLE_LIMIT, + quicksearch: '', + order_by: '', + is_desc: false + }) + let rowCount: number | undefined + try { + rowCount = await ops.getCount({ quicksearch: '' }) + } catch { + // count is best-effort — the sample rows are the important part + } + const columns = colDefs + .filter((c) => c.field) + .map((c) => ({ field: c.field as string, datatype: (c as any).datatype })) + return { ...base, columns, rows: rows.slice(0, SAMPLE_LIMIT), rowCount } + } catch (e) { + return { ...base, error: e instanceof Error ? e.message : String(e) } + } +} diff --git a/frontend/src/lib/components/recording/pipelineRecording.svelte.ts b/frontend/src/lib/components/recording/pipelineRecording.svelte.ts new file mode 100644 index 0000000000..557a3636ec --- /dev/null +++ b/frontend/src/lib/components/recording/pipelineRecording.svelte.ts @@ -0,0 +1,335 @@ +import { JobService, ScriptService, type Job } from '$lib/gen' +import type { AssetGraphResponse } from '$lib/components/assets/AssetGraph/types' +import { runBoundedCascade } from '$lib/components/assets/AssetGraph/cascadeRun' +import type { + CascadeNodeState, + CascadeRunResult +} from '$lib/components/assets/AssetGraph/cascadeOrchestrator' +import { truncateUuids } from './flowRecording.svelte' +import { capturePipelineAssetSample } from './pipelineAssetSample' +import type { + PipelineAssetSample, + PipelineRecordedCode, + PipelineRecording, + PipelineTimelineFrame, + RecordedJob, + RecordedNodeState +} from './types' + +/** + * Recorder for a data-pipeline cascade run. Unlike the flow/script recorders + * there is no single root job streaming sub-jobs over SSE — a pipeline run is a + * cascade of independent script jobs launched client-side and polled to + * completion. So this store captures two things: + * + * 1. the resolved asset graph (rendered read-only by the player), and + * 2. a timeline of per-node status snapshots (from the cascade orchestrator's + * `onUpdate`), each node mapped to its job id. + * + * For each launched node it opens the job's own SSE stream (`watchJob`) to + * capture incremental logs/result, storing them in the shared `RecordedJob` + * shape so the player can replay each node's details through the same + * `JobLoader` replay path the flow/script players use. + */ +export function createPipelineRecording(): PipelineRecordingStore { + let active = $state(false) + let startTime = 0 + let folder = '' + let graph: AssetGraphResponse | undefined = undefined + let timeline: PipelineTimelineFrame[] = [] + let jobs: Record = {} + let assetSamples: Record = {} + let codes: Record = {} + let watchedJobs = new Set() + let jobSources: EventSource[] = [] + + function closeSources() { + jobSources.forEach((es) => es.close()) + jobSources = [] + watchedJobs.clear() + } + + return { + get active() { + return active + }, + start(f: string, g: AssetGraphResponse) { + closeSources() + active = true + startTime = Date.now() + folder = f + // JSON round-trip to strip reactive proxies / non-serializable props. + graph = JSON.parse(JSON.stringify(g)) as AssetGraphResponse + timeline = [] + jobs = {} + assetSamples = {} + codes = {} + }, + /** Push a cascade status snapshot. Deep-cloned so a later mutation of the + * orchestrator's map can't rewrite an already-captured frame. */ + recordStatuses(statuses: Map) { + if (!active) return + const snapshot: Record = {} + for (const [path, st] of statuses) { + snapshot[path] = { status: st.status, jobId: st.jobId, error: st.error } + } + timeline.push({ t: Date.now() - startTime, statuses: snapshot }) + }, + /** Watch a launched node's SSE stream to capture its incremental + * logs/result. Mirrors flowRecording.watchSubJob's log-offset dedup. */ + watchJob(jobId: string, workspace: string) { + if (!active || watchedJobs.has(jobId)) return + watchedJobs.add(jobId) + + let logOffset = 0 + const params = new URLSearchParams({ + log_offset: '0', + running: 'true', + fast: 'true' + }) + const url = `/api/w/${workspace}/jobs_u/getupdate_sse/${jobId}?${params}` + const es = new EventSource(url) + jobSources.push(es) + + es.onmessage = (event) => { + try { + const data = JSON.parse(event.data) + if (data.type === 'ping' || data.type === 'timeout') return + if (data.type === 'error' || data.type === 'not_found') { + es.close() + return + } + if (!active) { + es.close() + return + } + + // Deduplicate log data: SSE may resend full log dumps on reconnect. + if (data.new_logs != null && data.log_offset != null) { + if (logOffset > 0 && data.log_offset <= logOffset) { + delete data.new_logs + delete data.log_offset + } else { + logOffset = data.log_offset + } + } else if (data.log_offset != null && data.log_offset > logOffset) { + logOffset = data.log_offset + } + + if (!jobs[jobId]) { + jobs[jobId] = { + initial_job: data.job ? (data.job as Job) : ({ id: jobId } as Job), + events: [] + } + } + jobs[jobId].events.push({ + t: Date.now() - startTime, + data + }) + if (data.completed) { + es.close() + } + } catch { + // Ignore parse errors + } + } + es.onerror = () => { + es.close() + } + }, + /** Fill in a node's completed job (fallback for anything the SSE stream + * missed — e.g. a job that finished before its stream was opened). + * Callable after stop() so late-fetched completed jobs still attach. */ + addCompletedJob(jobId: string, completedJob: Job) { + const snapshotJob = $state.snapshot(completedJob) as Job + if (!jobs[jobId]) { + jobs[jobId] = { initial_job: snapshotJob, events: [] } + } else if (!jobs[jobId].initial_job?.id) { + jobs[jobId].initial_job = snapshotJob + } + const hasCompleted = jobs[jobId].events.some((e) => e.data.completed) + if (!hasCompleted) { + jobs[jobId].events.push({ + t: Date.now() - startTime, + data: { completed: true, job: snapshotJob } + }) + } + }, + /** Attach a captured asset data-sample (called during finalize, after + * the run, for each ducklake/datatable asset). Keyed by `${kind}:${path}`. + * Callable after stop() so late captures still attach to the returned + * recording (which references the same `assetSamples` object). */ + recordAssetSample(sample: PipelineAssetSample) { + assetSamples[`${sample.kind}:${sample.path}`] = sample + }, + /** Attach a runnable's source (called during finalize, per script path). + * Callable after stop() so late captures still attach to the returned + * recording (which references the same `codes` object). */ + recordCode(path: string, code: PipelineRecordedCode) { + codes[path] = code + }, + stop(): PipelineRecording { + active = false + closeSources() + return { + version: 1, + type: 'pipeline', + recorded_at: new Date().toISOString(), + folder, + total_duration_ms: Date.now() - startTime, + graph: graph ?? ({ assets: [], runnables: [], edges: [], triggers: [] } as any), + timeline, + jobs, + assetSamples, + codes + } + }, + download(recording: PipelineRecording) { + const blob = new Blob([truncateUuids(JSON.stringify(recording, null, 2))], { + type: 'application/json' + }) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = `pipeline-recording-${(recording.folder || 'untitled').replace(/\//g, '-')}-${Date.now()}.json` + a.click() + URL.revokeObjectURL(url) + } + } +} + +export type PipelineRecordingStore = { + readonly active: boolean + start(folder: string, graph: AssetGraphResponse): void + recordStatuses(statuses: Map): void + watchJob(jobId: string, workspace: string): void + addCompletedJob(jobId: string, completedJob: Job): void + recordAssetSample(sample: PipelineAssetSample): void + recordCode(path: string, code: PipelineRecordedCode): void + stop(): PipelineRecording + download(recording: PipelineRecording): void +} + +// Max asset samples in flight during finalize — each is several preview jobs. +const ASSET_SAMPLE_CONCURRENCY = 4 + +/** Run `fn` over `items` at most `limit` at a time (sequential batches). */ +async function forEachWithConcurrency( + items: T[], + limit: number, + fn: (item: T) => Promise +): Promise { + for (let i = 0; i < items.length; i += limit) { + await Promise.all(items.slice(i, i + limit).map(fn)) + } +} + +/** + * Stop the recorder and enrich the recording with data the live SSE streams + * can't guarantee: each node's completed job (a fast job may finish before its + * stream opens), a data-sample per ducklake/datatable asset (offline table + * preview), and each step's source (by the exact hash that ran). Every fetch is + * best-effort — a step we can't resolve just replays with less detail. Shared by + * the pipeline editor's recorder and deploy-to-hub so both produce identical + * recordings. + */ +export async function finalizePipelineRecording( + store: PipelineRecordingStore, + workspace: string | undefined +): Promise { + const rec = store.stop() + if (!workspace) return rec + const ws = workspace + const jobIds = new Set() + for (const frame of rec.timeline) { + for (const st of Object.values(frame.statuses)) { + if (st.jobId) jobIds.add(st.jobId) + } + } + await Promise.all( + [...jobIds].map(async (jobId) => { + if (rec.jobs[jobId]?.events.some((e) => e.data.completed)) return + try { + const j = await JobService.getJob({ workspace: ws, id: jobId }) + store.addCompletedJob(jobId, j) + } catch { + // best-effort — a job we can't fetch just replays from its stream + } + }) + ) + // Each asset sample runs a metadata scan + a SELECT + a COUNT preview job, so a + // wide pipeline could fan out hundreds of jobs at once. Bound the concurrency + // to keep the recorder from saturating the worker pool. + const sampleTargets = (rec.graph.assets ?? []).filter( + (a) => a.kind === 'ducklake' || a.kind === 'datatable' + ) + await forEachWithConcurrency(sampleTargets, ASSET_SAMPLE_CONCURRENCY, async (a) => { + const sample = await capturePipelineAssetSample(ws, a.kind, a.path) + store.recordAssetSample(sample) + }) + const codeByPath = new Map() + for (const r of Object.values(rec.jobs)) { + const j = r.events.find((e) => e.data.completed)?.data.job as + | { job_kind?: string; script_path?: string; script_hash?: string } + | undefined + if (j?.job_kind === 'script' && j.script_path && j.script_hash) { + codeByPath.set(j.script_path, j.script_hash) + } + } + await Promise.all( + [...codeByPath].map(async ([path, hash]) => { + try { + const s = await ScriptService.getScriptByHash({ workspace: ws, hash }) + store.recordCode(path, { content: s.content, language: s.language }) + } catch { + // best-effort — a step we can't fetch just has no code in the player + } + }) + ) + return rec +} + +/** + * Run a folder's pipeline cascade end-to-end and capture it into a + * PipelineRecording — the self-contained path used by deploy-to-hub, where + * there is no editor page orchestrating the run. `launch`/`waitTerminal` are + * supplied by the caller (deployed-only launch, poll-based wait); this wires + * status/job capture around them and finalizes. + */ +export async function capturePipelineRecording(opts: { + workspace: string + folder: string + graph: AssetGraphResponse + scriptPaths: Set + launch: (path: string) => Promise + waitTerminal: (jobId: string) => Promise<'success' | 'failure'> + onUpdate?: (statuses: Map) => void +}): Promise<{ recording: PipelineRecording; result: CascadeRunResult & { cyclic: string[] } }> { + const store = createPipelineRecording() + store.start(opts.folder, opts.graph) + let result: CascadeRunResult & { cyclic: string[] } + try { + result = await runBoundedCascade({ + graph: opts.graph, + scripts: opts.scriptPaths, + launch: async (path) => { + const jobId = await opts.launch(path) + // No-op unless the store is active; captures the node's stream. + store.watchJob(jobId, opts.workspace) + return jobId + }, + waitTerminal: opts.waitTerminal, + onUpdate: (statuses) => { + store.recordStatuses(statuses) + opts.onUpdate?.(statuses) + } + }) + } catch (e) { + // The cascade threw before finalize could `stop()` the store: close the + // per-node SSE streams `watchJob` opened so they don't dangle. + store.stop() + throw e + } + const recording = await finalizePipelineRecording(store, opts.workspace) + return { recording, result } +} diff --git a/frontend/src/lib/components/recording/types.ts b/frontend/src/lib/components/recording/types.ts index 8d1dc8ca27..c47c9dc4e4 100644 --- a/frontend/src/lib/components/recording/types.ts +++ b/frontend/src/lib/components/recording/types.ts @@ -1,4 +1,5 @@ -import type { Job, OpenFlow } from '$lib/gen' +import type { AssetKind, Job, OpenFlow } from '$lib/gen' +import type { AssetGraphResponse } from '$lib/components/assets/AssetGraph/types' export type RecordedEvent = { t: number @@ -33,6 +34,69 @@ export type ScriptRecording = { job: RecordedJob } +/** Per-node status inside a recorded cascade frame (mirror of + * cascadeOrchestrator.CascadeNodeState, kept structurally independent so the + * recording module doesn't depend on the orchestrator internals). */ +export type RecordedNodeState = { + status: 'pending' | 'running' | 'success' | 'failure' | 'skipped' + jobId?: string + error?: string +} + +/** One frame of the cascade timeline: the full per-path status snapshot at + * `t` ms since the run started. Replaying these in order reproduces the graph + * animation (nodes lighting up / turning green/red) a live run would show. */ +export type PipelineTimelineFrame = { + t: number + statuses: Record +} + +/** A captured data-sample of a pipeline asset (ducklake table / datatable), + * so the player can show what an asset held after the run — offline, without + * re-querying the backend. Keyed in `assetSamples` by `${kind}:${path}`. */ +export type PipelineAssetSample = { + kind: AssetKind + path: string + /** Full asset URI, e.g. `ducklake://main/orders`. */ + uri: string + /** Column names (in order) of the sampled table. */ + columns: { field: string; datatype?: string }[] + /** Sampled rows (capped), each a record keyed by column field. */ + rows: unknown[] + /** Total row count if it could be fetched. */ + rowCount?: number + /** Set when the sample couldn't be captured (table missing, unsupported…). */ + error?: string +} + +export type PipelineRecording = { + version: 1 + type: 'pipeline' + recorded_at: string + folder: string + total_duration_ms: number + /** The resolved asset graph rendered read-only by the player. */ + graph: AssetGraphResponse + /** Ordered cascade status snapshots driving the node animation. */ + timeline: PipelineTimelineFrame[] + /** Per-node job streams (initial job + SSE events), keyed by job id, so the + * player can replay each node's logs/result/args offline via JobLoader. */ + jobs: Record + /** Per-asset data samples captured after the run, keyed by `${kind}:${path}`, + * so asset nodes are inspectable offline in the player. */ + assetSamples?: Record + /** Source code of each runnable, keyed by script path, captured at record + * time so the player can show a step's code offline. Absent for recordings + * taken before code capture existed (the player degrades gracefully). */ + codes?: Record +} + +/** A pipeline step's captured source. */ +export type PipelineRecordedCode = { + content: string + language: string +} + /** Minimal interface that both flow and script recording stores implement */ export interface ActiveRecording { recordInitialJob(jobId: string, job: Job): void diff --git a/frontend/src/lib/components/runs/TimeframeSelect.svelte b/frontend/src/lib/components/runs/TimeframeSelect.svelte index cc5a99f70c..f8d2b0df71 100644 --- a/frontend/src/lib/components/runs/TimeframeSelect.svelte +++ b/frontend/src/lib/components/runs/TimeframeSelect.svelte @@ -78,6 +78,8 @@ + + +{#snippet presetButtons()} + {#each items as item (item.label)} + + {/each} +{/snippet} +
- {/each} + {#if isSmall} +
+
+ {@render presetButtons()} +
+
+ + {#snippet children({ item })} + + + {/snippet} + + range, + (v) => + onManualInput( + smallBound === 'end' + ? { maxTs: fromCalendarDate(v.end)?.toISOString() ?? null } + : { minTs: fromCalendarDate(v.start)?.toISOString() ?? null } + ) + } + /> +
- range, - (v) => onManualInput({ minTs: fromCalendarDate(v.start)?.toISOString() ?? null }) - } - /> - range, - (v) => onManualInput({ maxTs: fromCalendarDate(v.end)?.toISOString() ?? null }) - } - /> -
+ {:else} +
+
+ {@render presetButtons()} +
+ range, + (v) => onManualInput({ minTs: fromCalendarDate(v.start)?.toISOString() ?? null }) + } + /> + range, + (v) => onManualInput({ maxTs: fromCalendarDate(v.end)?.toISOString() ?? null }) + } + /> +
+ {/if} {/snippet} diff --git a/frontend/src/lib/components/scriptSettings.test.ts b/frontend/src/lib/components/scriptSettings.test.ts new file mode 100644 index 0000000000..f010e3863a --- /dev/null +++ b/frontend/src/lib/components/scriptSettings.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect } from 'vitest' +import { getActiveScriptSettingsBadges } from './scriptSettings' + +describe('getActiveScriptSettingsBadges', () => { + it('returns no badges for undefined or empty settings', () => { + expect(getActiveScriptSettingsBadges(undefined)).toEqual([]) + expect(getActiveScriptSettingsBadges({})).toEqual([]) + }) + + it('only surfaces settings that are actually active', () => { + const keys = getActiveScriptSettingsBadges({ + concurrent_limit: 3, + concurrency_time_window_s: 60, + cache_ttl: 600, + timeout: 120, + priority: 50, + tag: 'gpu' + }).map((b) => b.key) + expect(keys).toEqual(['concurrency', 'cache', 'timeout', 'priority', 'tag']) + }) + + it('treats a zero/absent priority and non-positive debounce as inactive', () => { + const keys = getActiveScriptSettingsBadges({ + priority: 0, + debounce_delay_s: 0 + }).map((b) => b.key) + expect(keys).toEqual([]) + }) + + it('treats non-positive concurrency limits, timeouts and cache ttl as inactive (legacy zero rows)', () => { + const keys = getActiveScriptSettingsBadges({ + concurrent_limit: 0, + timeout: 0, + cache_ttl: 0 + }).map((b) => b.key) + expect(keys).toEqual([]) + }) + + it('keeps delete_after_secs of 0 active (immediate deletion is a real setting)', () => { + const badge = getActiveScriptSettingsBadges({ delete_after_secs: 0 }) + expect(badge.map((b) => b.key)).toEqual(['delete_after_use']) + expect(badge[0].detail).toContain('immediately') + }) + + it('pluralizes the concurrency detail correctly', () => { + expect(getActiveScriptSettingsBadges({ concurrent_limit: 1 })[0].detail).toContain( + 'Max 1 execution' + ) + expect(getActiveScriptSettingsBadges({ concurrent_limit: 2 })[0].detail).toContain( + 'Max 2 executions' + ) + }) +}) diff --git a/frontend/src/lib/components/scriptSettings.ts b/frontend/src/lib/components/scriptSettings.ts new file mode 100644 index 0000000000..4a3b3ad5b8 --- /dev/null +++ b/frontend/src/lib/components/scriptSettings.ts @@ -0,0 +1,137 @@ +import { + Gauge, + Database, + Timer, + Hourglass, + Repeat, + Cpu, + Trash2, + ChevronsUp, + Tag +} from 'lucide-svelte' +import type { ScriptLang } from '$lib/gen' + +// Subset of Script/NewScript fields that make up the "advanced runtime settings" +// surfaced both in the standalone script editor and, via the mini settings drawer, +// from within the flow editor for workspace-script steps. +export type ScriptAdvancedSettingsFields = { + path?: string + language?: ScriptLang + schema?: unknown + tag?: string + concurrent_limit?: number + concurrency_time_window_s?: number + concurrency_key?: string + cache_ttl?: number + cache_ignore_s3_path?: boolean + timeout?: number + debounce_delay_s?: number + debounce_key?: string + debounce_args_to_accumulate?: string[] + max_total_debouncing_time?: number + max_total_debounces_amount?: number + restart_unless_cancelled?: boolean + dedicated_worker?: boolean + delete_after_secs?: number + priority?: number +} + +export type ScriptSettingsBadge = { + key: string + label: string + icon: any + detail: string +} + +// Compute the list of active advanced settings for a script, used to render +// at-a-glance badges in the editor top bar and in the flow drawers. +export function getActiveScriptSettingsBadges( + settings: ScriptAdvancedSettingsFields | undefined +): ScriptSettingsBadge[] { + if (!settings) return [] + const badges: ScriptSettingsBadge[] = [] + // Non-positive concurrent_limit / timeout are treated as unset by the runtime + // (legacy zero rows), so don't surface them as active settings. + if (settings.concurrent_limit != undefined && settings.concurrent_limit > 0) { + badges.push({ + key: 'concurrency', + label: 'Concurrency', + icon: Gauge, + detail: `Max ${settings.concurrent_limit} execution${ + settings.concurrent_limit === 1 ? '' : 's' + }${ + settings.concurrency_time_window_s != undefined + ? ` / ${settings.concurrency_time_window_s}s` + : '' + }` + }) + } + if (settings.cache_ttl != undefined && settings.cache_ttl > 0) { + badges.push({ + key: 'cache', + label: 'Cache', + icon: Database, + detail: `Cached for ${settings.cache_ttl}s` + }) + } + if (settings.timeout != undefined && settings.timeout > 0) { + badges.push({ + key: 'timeout', + label: 'Timeout', + icon: Timer, + detail: `${settings.timeout}s` + }) + } + if (settings.debounce_delay_s != undefined && settings.debounce_delay_s > 0) { + badges.push({ + key: 'debounce', + label: 'Debounce', + icon: Hourglass, + detail: `Debounced by ${settings.debounce_delay_s}s` + }) + } + if (settings.restart_unless_cancelled) { + badges.push({ + key: 'perpetual', + label: 'Perpetual', + icon: Repeat, + detail: 'Restarts unless cancelled' + }) + } + if (settings.dedicated_worker) { + badges.push({ + key: 'dedicated', + label: 'Dedicated', + icon: Cpu, + detail: 'Runs on dedicated workers' + }) + } + if (settings.delete_after_secs != undefined) { + badges.push({ + key: 'delete_after_use', + label: 'Delete after use', + icon: Trash2, + detail: + settings.delete_after_secs === 0 + ? 'Deleted immediately after completion' + : `Deleted ${settings.delete_after_secs}s after completion` + }) + } + if (settings.priority != undefined && settings.priority > 0) { + badges.push({ + key: 'priority', + label: 'High priority', + icon: ChevronsUp, + detail: `Priority ${settings.priority}` + }) + } + if (settings.tag) { + badges.push({ + key: 'tag', + label: settings.tag, + icon: Tag, + detail: `Worker tag: ${settings.tag}` + }) + } + return badges +} diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 3e631a55e9..d2b27635da 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -40,7 +40,15 @@ import { Alert } from '../common' import Popover from '../Popover.svelte' import Logs from 'lucide-svelte/icons/logs' - import { AwsIcon, AzureIcon, GoogleCloudIcon, KafkaIcon, MqttIcon, NatsIcon } from '../icons' + import { + AwsIcon, + AzureIcon, + GoogleCloudIcon, + KafkaIcon, + MqttIcon, + AmqpIcon, + NatsIcon + } from '../icons' import RunsSearch from './RunsSearch.svelte' import AskAiButton from '../copilot/AskAiButton.svelte' @@ -152,6 +160,13 @@ icon: MqttIcon, disabled: $userStore?.operator }, + { + search_id: 'nav:amqp_triggers', + label: 'Go to AMQP triggers', + action: (newtab: boolean = false) => gotoPage('/amqp_triggers', newtab), + icon: AmqpIcon, + disabled: $userStore?.operator + }, { search_id: 'nav:email_triggers', label: 'Go to Email triggers', diff --git a/frontend/src/lib/components/sessions/FlowEditorView.svelte b/frontend/src/lib/components/sessions/FlowEditorView.svelte index 845c6e2bdc..5977c802ec 100644 --- a/frontend/src/lib/components/sessions/FlowEditorView.svelte +++ b/frontend/src/lib/components/sessions/FlowEditorView.svelte @@ -77,7 +77,12 @@ {onNavigate} {isActiveSession} isActiveTab={active} - effectivePath={() => cell.store.val?.path ?? path} + effectivePath={() => + // A flow's typed rename lives in `draft_path` (`val.path` is the storage + // key), unlike scripts where `val.path` is the typed name — without it the + // live-draft registration hides a staged rename from lists and pickers. + ((cell.store.val as { draft_path?: string } | undefined)?.draft_path || cell.store.val?.path) ?? + path} > {#snippet editor()} + `workspace={workspaceId}` scopes every trigger backend call to THIS + session's (possibly forked) workspace — a session never switches the global + `$workspaceStore` (SessionPicker), so without this the editors would write + to the nav workspace. --> graphRes.refetch()} /> diff --git a/frontend/src/lib/components/sessions/PreviewRouterPicker.svelte b/frontend/src/lib/components/sessions/PreviewRouterPicker.svelte index 80d09b6004..58647481fe 100644 --- a/frontend/src/lib/components/sessions/PreviewRouterPicker.svelte +++ b/frontend/src/lib/components/sessions/PreviewRouterPicker.svelte @@ -1,12 +1,14 @@ {#snippet leafIcon(leaf: DrillLeaf)} @@ -163,10 +237,7 @@ only ever navigates between items) doesn't grow a Pages section. {/snippet} {#snippet branchIcon(branch: DrillBranch)} - {#if branch.key === 'kind:flow' || branch.key === 'kind:script' || branch.key === 'kind:app'} - {@const k = branch.key.slice(5) as Kind} - - {:else if branch.icon} + {#if branch.icon} {@const Icon = branch.icon} {/if} @@ -184,9 +255,12 @@ only ever navigates between items) doesn't grow a Pages section. {leafIcon} {branchIcon} leafSecondary={(leaf, scope) => - leaf.data.type === 'item' ? relativizeWorkspacePath(leaf.data.item.path, scope) : undefined} + leaf.data.type === 'item' + ? relativizeWorkspacePath(workspaceItemDisplayPath(leaf.data.item), scope) + : undefined} onScopeChange={(scope) => { if (scope.length > 0) loader.ensureForScopeSegment(scope[0]) }} onFilterChange={loader.onFilterChange} + {rootLoading} /> diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index 7ee4b02b12..eadbc22293 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -8,12 +8,10 @@ type SessionPreviewTab } from './sessionState.svelte' import type { SessionRuntime } from './sessionRuntime.svelte' + import { Loader2 } from 'lucide-svelte' import { resolvePreviewTab, parsePreviewItemRoute } from './previewRouter' import { withMenuHidden } from './sessionMode.svelte' - import ScriptEditorView from './ScriptEditorView.svelte' - import FlowEditorView from './FlowEditorView.svelte' - import RawAppEditorView from './RawAppEditorView.svelte' - import PipelineEditorView from './PipelineEditorView.svelte' + import ArtifactViewer from '../copilot/chat/artifacts/ArtifactViewer.svelte' let { tab, @@ -57,6 +55,13 @@ ) const isActiveSession = $derived(!!session && sessionState.currentSessionId === session.id) + // Resolved live from the session's store so an update_artifact re-renders the panel. + const artifact = $derived( + slot.kind === 'artifact' + ? runtime?.manager.artifacts.artifacts.find((a) => a.id === slot.id) + : undefined + ) + let frame: HTMLIFrameElement | undefined = $state() // Pages whose theme we mirror on live toggles. Regular apps are the only item @@ -94,9 +99,14 @@ // in-frame navigation may have stripped of ?workspace= — booting the frame // into the top-level navigation workspace instead of the session fork // (sessionStorage/localStorage are shared with the top window, so the - // scoping can only live in the URL). replace() forces the load even when - // the target equals the current URL. - win.location.replace(withMenuHidden(tab.loc || tab.url, workspaceId || undefined)) + // scoping can only live in the URL). But replace() to the frame's exact + // current URL is a no-op when it carries a fragment (same-document + // navigation, no load) — only then fall back to location.reload(), which + // always performs a full load of that same URL. + const target = withMenuHidden(tab.loc || tab.url, workspaceId || undefined) + const { pathname, search, hash } = win.location + if (pathname + search + hash === target) win.location.reload() + else win.location.replace(target) } catch { // Cross-navigation timing — skip; the next mutation reloads again. } @@ -105,42 +115,111 @@ const visibility = $derived( active ? 'z-10 opacity-100 pointer-events-auto' : 'z-0 opacity-0 pointer-events-none' ) + + let flashing = $state(false) + let flashTimer: ReturnType | undefined + // Guard against the effect's non-pulse reruns (tab/runtime changes) firing a flash. + let lastPulseNonce = -1 + $effect(() => { + const pulse = runtime?.previewTabs.focusPulse + if (!pulse || pulse.nonce === lastPulseNonce) return + lastPulseNonce = pulse.nonce + if (pulse.id !== tab.id) return + flashing = true + clearTimeout(flashTimer) + flashTimer = setTimeout(() => (flashing = false), 800) + }) + $effect(() => () => clearTimeout(flashTimer)) + + // Forced-load signal for a navigation to the tab's exact current URL (see + // pulseReload) — without it the page never re-runs its URL-driven behavior. + // Seeded from the current nonce: a pulse from before this host mounted is + // already satisfied by the initial iframe load. + let lastReloadNonce = runtime?.previewTabs.reloadPulse.nonce ?? -1 + $effect(() => { + const pulse = runtime?.previewTabs.reloadPulse + if (!pulse || pulse.nonce === lastReloadNonce) return + lastReloadNonce = pulse.nonce + if (pulse.id !== tab.id) return + reload() + }) +{#snippet editorLoading()} +
+ +
+{/snippet} + {#if slot.kind === 'editor' && mounted && runtime}
+ {#if slot.editorKind === 'flow'} - + {#await import('./FlowEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else if slot.editorKind === 'script'} - + {#await import('./ScriptEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else if slot.editorKind === 'pipeline'} - + {#await import('./PipelineEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {:else} - + {#await import('./RawAppEditorView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} {/if}
+{:else if slot.kind === 'artifact' && mounted} +
+ {#if artifact} + + {:else if !runtime?.manager.artifacts.loading} +
This artifact is no longer available.
+ {/if} + + +
{:else if mounted} ` + try { + await navigator.clipboard.writeText(snippet) + sendUserToast('Iframe snippet copied to clipboard') + } catch { + sendUserToast('Failed to copy snippet', true) + } + } + + +{#if deployHub.session} + {#key deployHub.session} + {@const s = deployHub.session} +
+ + {#snippet header()} + {@const stepNum = + s.phase === 'predeploy' + ? 1 + : s.phase === 'draft' + ? 2 + : s.phase === 'under_review' + ? 3 + : 4} +
+
    + + How to publish your project to the Hub + +
  1. 1 ? 'opacity-60' : ''}> + {stepNum > 1 ? '✓' : '1.'} + Bundle your project — creates a draft + on the Hub with every selected script, flow, app and resource from this folder. +
  2. +
  3. 2 ? 'opacity-60' : 'opacity-40'} + > + {stepNum > 2 ? '✓' : '2.'} + Generate iframes & recordings — share + public apps as iframes, capture one execution per script/flow, and record the whole data-pipeline + cascade as one interactive replay. +
  4. +
  5. 3 ? 'opacity-60' : 'opacity-40'} + > + {stepNum > 3 ? '✓' : '3.'} + Submit for review — send the bundle for + approval. +
  6. +
+
+ {#if s.phase === 'predeploy'} + + Step 1: Bundle your project + + {:else if s.phase === 'draft'} + + Step 2: Generate iframes & recordings + + {:else if s.phase === 'under_review'} + Step 3: Awaiting review + {:else} + Live on the Hub + {/if} + {#if s.phase !== 'predeploy'} + + + on Hub: + {s.hubName || s.hubSlug} + + + Open in Hub + + {/if} +
+ {#if s.phase === 'predeploy'} + + {:else if s.phase === 'draft'} + + {:else if s.phase === 'under_review'} + + {/if} +
+
+ {#if s.phase === 'predeploy'} +
+ + Bundling creates a draft project on the Hub from the selected scripts, flows and + apps of {s.selectedFolder}/. + {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected. + +
+
+ + Resource dependencies + {#if s.detectingResources} + + {:else} + ({s.dependencyTypes.length}) + {/if} + + Resource types the selected items depend on (whether passed as inputs or + referenced by a hardcoded path). Synced to the Hub so a fork knows what + credentials it needs to fill. + + + {#if s.dependencyTypes.length === 0} + + No resource references detected in the current selection. + + {:else} + {#each s.dependencyTypes as r (r.resource_type)} + + {r.resource_type} + + {/each} + + {/if} +
+
+ + Data table dependencies + {#if s.detectingDatatables} + + {:else} + ({s.datatableUsage.size}) + {/if} + + Data tables the selected items read or write. A best-effort CREATE TABLE + migration for these is generated in the bundle step and shipped with the + project, so a fork can recreate the tables it needs. + + + {#if s.datatableUsage.size === 0} + + No data table usage detected in the current selection. + + {:else} + {#each [...s.datatableUsage] as [dt, tables] (dt)} + + {dt} + {#if tables.size > 0} + ×{tables.size} + {/if} + + {/each} + {/if} +
+ {/if} + {#if s.phase === 'draft'} +
+ + A recording captures one real run of a script or flow — inputs, logs, step outputs + and result — replayable on the Hub so visitors see it work before forking. Public + apps can also be shared as live iframes. Optional, but recommended. + +
+ {/if} + {#if s.phase === 'draft' && s.isPipelineProject} +
+
+ + Data pipeline recording + {#if s.pipelineRecorded} + + Recorded + + {/if} +
+ + {#if s.pipelineRecordingResult} + + + {/if} +
+
+ + Runs this project's {s.selectedFolder}/ pipeline + cascade ({s.recordablePipelineScriptPaths.length} step{s + .recordablePipelineScriptPaths.length === 1 + ? '' + : 's'}) and captures the asset graph, per-step logs/results and table samples + into one interactive replay for the project page. + + {#if s.pipelineRunState === 'running'} +
+ Running the pipeline cascade… +
+ {:else if s.pipelineRunState === 'success'} +
+ Cascade succeeded — preview it, then save as the recording. +
+ {:else if s.pipelineRunState === 'failed'} +
+ + {s.pipelineRunError ?? 'Cascade failed'} +
+ {/if} +
+ {/if} + {#if s.phase === 'predeploy'} +
+ + Triggers + {#if s.triggersLoading} + + {:else} + ({s.relevantTriggers.length}) + {/if} + + {#if s.triggerDiscoveryFailed} + + Some trigger kinds could not be listed — publishing is disabled so triggers + aren't silently left out of the bundle. + + + {:else if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} + + {TRIGGER_KINDS[kind].badge} + ×{triggers.length} + + {/each} + + {/if} +
+ {/if} + {#if s.phase === 'under_review'} +
+ +
+ Locked while under review + + The Windmill team is reviewing this submission. Editing, recording, and sharing + actions are disabled. Estimated turnaround: 1-2 business days. + +
+
+ {/if} + {#if s.phase === 'draft'} + {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} + {@const pct = s.recordableItems.length + ? Math.round((recordedCount / s.recordableItems.length) * 100) + : 0} +
+ {recordedCount}/{s.recordableItems.length} +
+
+
+ + {s.allRecorded ? 'Full recordings' : 'Recordings recommended'} + +
+ {/if} + {#if s.phase === 'predeploy' && s.isPipelineProject} +
+ + {#if pipelineGraphOpen} +
+ + The {s.selectedFolder}/ scripts and the data tables + they read and write, as a single pipeline. This whole cascade can be captured as + an interactive replay once the project is bundled. + + {#if s.pipelineGraph} +
+ +
+ {:else} + Loading pipeline graph… + {/if} +
+ {/if} +
+ {/if} +
+ {/snippet} + + {#snippet itemSummary(item)} + {@const it = item as DeployItem} + + + {it.summary?.trim() || it.path} + + {#if it.kind === 'script' && s.pipelineScriptPathSet.has(it.path)} + + Pipeline + + {/if} + + {/snippet} + + {#snippet itemActions(item)} + {@const it = item as DeployItem} + {#if s.phase !== 'predeploy' && canRecord(it.kind)} + {#if it.rec === 'recorded'} + + Recorded + + {#if s.recordings[it.key]} + + See recording + + {/if} + {#if s.phase === 'draft'} + + {/if} + {:else if s.phase === 'draft'} + No recording + + {:else} + No recording + {/if} + {/if} + {#if s.phase !== 'predeploy' && canShareAsIframe(it)} + {#if it.published} + + Public + + {#if it.publicUrl} + + Open + + + {:else if s.phase !== 'under_review'} + + + {/if} + {#if s.phase !== 'under_review'} + + {/if} + {:else if s.phase !== 'under_review'} + + {/if} + {/if} + {/snippet} + + {#snippet footer()} +
+ {#if s.phase === 'predeploy'} + + Select the items to include — all selected by default. + + {:else if s.phase === 'draft'} + + {#if s.allRecorded} + All scripts and flows have a recording — best chance of approval and featuring. + {:else} + {s.recordableItems.filter((i) => i.rec === 'recorded').length} of {s + .recordableItems.length} + recorded. Bundles with full recordings get approved faster and featured on the public + Hub. + {/if} + + {:else if s.phase === 'under_review'} + + Waiting for the Windmill team to review the submission. + + {:else} + Iterate further by starting a new draft. + {/if} +
+ {/snippet} +
+
+ + + recordDrawer?.closeDrawer()} + > +
+

+ Run this {s.recordTarget?.kind} once with the inputs below. The full execution — args, logs, + intermediate step outputs and final result — is saved as a replayable recording + shown on the Hub page. Visitors can step through it to see how the {s.recordTarget + ?.kind} works without running anything themselves. +

+ + {#if s.runState !== 'idle'} +
+
+ {#if s.runState === 'running'} + + Running… + {:else if s.runState === 'success'} + + + Execution succeeded + + {:else} + + Execution failed + {/if} + {#if s.runJobId} + + Open job + + {/if} +
+ {#if s.runState === 'success' && s.runResult !== undefined} +
+ Result preview: +
{JSON.stringify(s.runResult, null, 2)}
+
+ {:else if s.runState === 'failed' && s.runError} +
{s.runError}
+ {/if} + {#if s.runState === 'success'} +
+ + Looks good? Save this run as the Hub recording. + + +
+ {:else if s.runState === 'failed'} + + Fix inputs and try again. Only successful runs can be saved as a recording. + + {/if} +
+ {/if} + + {#if s.recordSchemaLoading} + Loading schema… + {:else} + + {/if} +
+ {#snippet actions()} + {#if s.runState === 'success'} + + + {:else} + + {/if} + {/snippet} +
+
+ + + pipelinePreviewDrawer?.closeDrawer()} + > + {#if s.pipelineRecordingResult} +
+ +
+ {/if} +
+
+ + + publishDrawer?.closeDrawer()} + > +
+

+ Expose {s.publishTarget?.path} at a public URL + so it can be embedded as an iframe (e.g. on the Hub, a docs page, or your own site). Anyone + with the URL will be able to interact with it. +

+ +
+
+ + Rate limit (workspace-wide) + + Caps public app executions per minute per server. Applies to all public apps in this + workspace. + +
+ {#if s.workspaceRateLimit && s.workspaceRateLimit > 0} + + Currently {s.workspaceRateLimit} executions + / minute / server. + + {:else} + + No rate limit configured — anyone with the URL can hit this app at any rate. + + {/if} + publishDrawer?.closeDrawer()} + > + Edit in Workspace settings → Apps + +
+
+ {#snippet actions()} + + + {/snippet} +
+
+ + + resourceDrawer?.closeDrawer()}> +
+

+ Resource types the selected items depend on. Each is synced to the Hub so a fork knows + what credentials it needs to fill. Input means the + item takes the resource as a parameter; + hardcoded path means the item pins a specific resource + path in its code. +

+ {#if s.dependencyTypes.length === 0} + No resource references in the current selection. + {:else} + {#each s.dependencyTypes as r (r.resource_type)} +
+
+ + {r.resource_type} + + + {r.usages.length} usage{r.usages.length > 1 ? 's' : ''} + +
+
+ {#each r.usages as u, ui (ui)} + {#if u.role === 'trigger'} +
+ + {u.label} + + {TRIGGER_KINDS[u.triggerKind].badge} trigger + +
+ {:else} + {@const itemUrl = s.itemUrl(u.kind, u.itemPath)} +
+
+ {#if u.kind === 'script'} + + {:else if u.kind === 'flow'} + + {:else} + + {/if} + {u.label} + + {u.role === 'hardcoded' ? 'hardcoded path' : 'input'} + {#if u.role === 'hardcoded'} + + + {#snippet text()} +
+ + This {u.kind} references the resource by a hardcoded path + $res:{u.path}. + + + For portability, prefer taking the resource as an input — a + fork won't have this exact path. It's relocated into the + project on publish, but converting it to an input keeps the + item reusable. + +
+ {/snippet} +
+ {/if} +
+ {#if itemUrl} + + + + {/if} +
+
+ {/if} + {/each} +
+
+ {/each} + {/if} +
+
+
+ + + triggerDrawer?.closeDrawer()}> +
+

+ Triggers attached to the selected scripts and flows. Synced to the Hub as + disabled stubs. Recipients review and enable each one + manually after importing. External hooks (Slack/Discord webhooks, message-queue + subscriptions, etc.) must be re-registered against the importing instance. +

+ {#if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} +
+
+ + {TRIGGER_KINDS[kind].badge} + + + {triggers.length} trigger{triggers.length > 1 ? 's' : ''} + + {#if TRIGGER_KINDS[kind].note} + + + + {#snippet text()} +
+ {TRIGGER_KINDS[kind].note} +
+ {/snippet} +
+
+ {/if} +
+
+ {#each triggers as t (t.path)} + {@const runnableSummary = s.runnableSummaryByPath.get( + `${t.is_flow ? 'flow' : 'script'}:${t.script_path}` + )} + {@const details = triggerDetails(t)} + {@const cfg = t.config as any} + {@const previewKey = + t.kind === 'schedule' ? `${cfg.schedule}|${cfg.timezone}` : ''} + {@const preview = + t.kind === 'schedule' ? s.schedulePreviews[previewKey] : undefined} + {@const triggerUrl = s.triggerListUrl(t.kind)} +
+
+ {#if t.is_flow} + + {:else} + + {/if} + + {runnableSummary || t.script_path} + + + {t.is_flow ? 'flow' : 'script'} + + {#if triggerUrl} + + + + {/if} +
+ {#if details.length > 0} +
+ {#each details as d (d.label)} +
{d.label}
+
{d.value}
+ {/each} + {#if t.kind === 'schedule'} +
Next runs
+
+ {#if preview && preview.length > 0} +
+ {#each preview as date (date)} + {displayDate(date)} + {/each} +
+ {:else if preview && preview.length === 0} + No upcoming run + {:else} + Loading… + {/if} +
+ {/if} +
+ {/if} +
+ {/each} +
+
+ {/each} + {/if} +
+
+
+ + + bundleDrawer?.closeDrawer()}> +
+

+ Name and document your bundle. The readme can be updated later, but a clear one speeds + up the Windmill team's review. +

+ {#if s.bundlePreview && s.bundlePreview.unresolved.length > 0} +
+ {s.bundlePreview.unresolved.length} unresolved reference(s) — cannot publish + + These items or resources couldn't be resolved, so the bundle would ship broken + references. Deselect or fix them, then retry: + +
    + {#each s.bundlePreview.unresolved as u (u)} +
  • {u}
  • + {/each} +
+
+ {/if} + +
+ Project slug + + {s.effectiveSlug || sanitizeSlug(s.hubName) || '—'} + + + {#if s.effectiveSlug} + Locked — items live under f/{s.effectiveSlug}/. + {:else if s.hubName.trim() && !isValidSlug(sanitizeSlug(s.hubName))} + + The name yields an invalid slug. Use at least 3 letters/digits. + + {:else} + Auto-generated from the name. Once project forked, items will live under + f/{sanitizeSlug(s.hubName) || ''}/. + {/if} + +
+ +
+ {s.hubLogo ? 'Preview' : 'Logo'} + {#if s.hubLogo} + + + This is how your project card will look on the Hub. + +
+
+
+ Project logo preview +
+
+
+ {s.hubName.trim() || 'Project name'} +
+

+ {s.hubSummary.trim() || s.hubName.trim() || 'Short one-liner shown on the Hub card'} +

+
+
+
+ + +
+
+ {:else} + {#if s.hubLogo === null} +
+ + The project's current logo will be removed when you publish. + + +
+ {:else if s.hubHasRemoteLogo} +
+ + This project already has a custom logo on the Hub. + + +
+ {/if} + + {/if} + + {#if s.hubLogo === undefined} + + Optional. Shown on the Hub project card and page. Leaving it empty keeps the + project's current logo. + + {/if} +
+ +
+
+ + Data table migrations +
+ {#if s.migrationsGenerating} +
+ + Detecting data tables used by this project… +
+ {:else if s.migrationDrafts.length === 0} + + No data table usage detected in this project's scripts, flows, or raw apps. + + {:else} + + We detected these data tables. When included, the migration recreates their tables + on import. Best-effort — review and edit before publishing. + + {#each s.migrationDrafts as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ +
+ {/each} + {/if} +
+
+ {#snippet actions()} + + {/snippet} +
+
+ {/key} +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte new file mode 100644 index 0000000000..76e3da3e07 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte @@ -0,0 +1,40 @@ + + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts new file mode 100644 index 0000000000..99d45a7007 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -0,0 +1,1845 @@ +import { untrack } from 'svelte' +import { base } from '$lib/base' +import { + AppService, + FlowService, + JobService, + RawAppService, + ResourceService, + ScriptService, + WorkspaceService, + ScheduleService +} from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { sleep, emptySchema } from '$lib/utils' +import { computeSecretUrl } from '$lib/components/apps/editor/appDeploy.svelte' +import { + buildProjectBundle, + buildPathMap, + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + rewriteTriggerConfig, + rewriteVarRefsInValue, + type BundleDeps, + type BundledItem, + type FetchedItem, + type ItemKind, + type ItemRef, + type ProjectBundle +} from './projectBundle' +import { + detectDatatableTables, + generateDatatableMigrations, + type GeneratedMigration +} from './projectMigrations' +import type { Kind } from '$lib/utils_deployable' +import type { AssetGraphResponse } from '$lib/components/assets/AssetGraph/types' +import { + CASCADE_JOB_TIMEOUT_MS, + CASCADE_POLL_INTERVAL_MS, + DATA_ASSET_KINDS +} from '$lib/components/assets/AssetGraph/cascadeRun' +import { capturePipelineRecording } from '$lib/components/recording/pipelineRecording.svelte' +import type { PipelineRecording } from '$lib/components/recording/types' +import { + TRIGGER_KINDS, + listAllWorkspaceTriggers, + triggerResourcePath, + triggerHandlerRefs, + portableTriggerConfig, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' + +export type Phase = 'predeploy' | 'draft' | 'under_review' | 'live' +export type RecStatus = 'none' | 'recorded' +export interface DeployItem { + key: string + path: string + kind: Kind + summary?: string + rec: RecStatus + published?: boolean + publicUrl?: string + [k: string]: unknown +} + +export const canRecord = (k: Kind) => k === 'script' || k === 'flow' +// Legacy raw apps live only in the `raw_app` table, but the iframe share flow +// drives AppService (the `app` table), so it can only target apps stored there. +export const canShareAsIframe = (it: DeployItem): boolean => + it.kind === 'app' || (it.kind === 'raw_app' && it.appTable === true) + +// Hub rehydration only carries draft membership, not the live share state of an +// app. Copy the public-execution flag, public URL, and app-table origin from the +// loaded workspace items onto matching draft items so a still-public app keeps its +// Public badge, Unpublish, and iframe controls after its draft is reopened. Returns +// the original array unchanged when nothing needs merging (stable reference). +export function mergeShareState( + draftItems: DeployItem[], + workspaceItems: DeployItem[] +): DeployItem[] { + if (draftItems.length === 0 || workspaceItems.length === 0) return draftItems + const byKey = new Map(workspaceItems.map((w) => [w.key, w])) + let changed = false + const merged = draftItems.map((d) => { + const w = byKey.get(d.key) + if (!w) return d + if (w.published !== d.published || w.publicUrl !== d.publicUrl || w.appTable !== d.appTable) { + changed = true + return { ...d, published: w.published, publicUrl: w.publicUrl, appTable: w.appTable } + } + return d + }) + return changed ? merged : draftItems +} + +export function sanitizeSlug(s: string): string { + return s + .toLowerCase() + .replace(/[_\s]+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50) + .replace(/-+$/g, '') +} +const SLUG_RE = /^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$/ +export function isValidSlug(s: string): boolean { + return SLUG_RE.test(s) +} + +export type RunState = 'idle' | 'running' | 'success' | 'failed' + +const ITEM_KIND_ROUTE: Record = { + script: 'scripts/get', + flow: 'flows/get', + app: 'apps/get', + raw_app: 'apps_raw/get' +} + +const HIDDEN_RESOURCE_TYPES = new Set(['app_theme', 'state', 'cache']) + +// Prune a folder's asset graph to a set of scripts so a pipeline recording only +// runs, renders and samples the project's included members — a deselected branch +// (its nodes, code, logs/results and table samples) never enters the recording. +// Assets kept are only those an included script touches; edges/triggers only +// those anchored on an included runnable. +function pruneGraphToScripts(graph: AssetGraphResponse, scripts: Set): AssetGraphResponse { + const runnables = graph.runnables.filter((r) => scripts.has(r.path)) + const edges = graph.edges.filter((e) => scripts.has(e.runnable_path)) + const keptAssets = new Set(edges.map((e) => `${e.asset_kind}:${e.asset_path}`)) + const assets = graph.assets.filter((a) => keptAssets.has(`${a.kind}:${a.path}`)) + const triggers = graph.triggers.filter((t) => scripts.has(t.runnable_path)) + const macro_edges = graph.macro_edges?.filter( + (m) => scripts.has(m.consumer_path) && scripts.has(m.lib_path) + ) + const test_edges = graph.test_edges?.filter( + (t) => scripts.has(t.runnable_path) && scripts.has(t.producer_path) + ) + return { assets, runnables, edges, triggers, macro_edges, test_edges } +} + +function typesFromSchema(schema: any): string[] { + const out = new Set() + const props = schema?.properties + if (props && typeof props === 'object') { + for (const key of Object.keys(props)) { + const fmt = props[key]?.format + if (typeof fmt === 'string' && fmt.startsWith('resource-')) { + out.add(fmt.slice('resource-'.length)) + } + } + } + return [...out] +} + +type DependencyUsage = + | { role: 'input'; label: string; kind: ItemKind; itemPath: string } + | { role: 'hardcoded'; label: string; kind: ItemKind; path: string; itemPath: string } + | { role: 'trigger'; label: string; triggerKind: WorkspaceTriggerKind; path: string } +export interface DependencyType { + resource_type: string + hasHardcoded: boolean + usages: DependencyUsage[] +} + +interface SessionDeps { + hasEeLicense: () => boolean +} + +/** + * All state and async operations for one Deploy-to-Hub surface, bound to an + * immutable (workspace, folder) pair. A workspace or folder change never mutates + * a session — `useDeployToHubSession` replaces the instance, so in-flight async + * work keeps writing to the discarded object and cannot leak into the new scope. + * The only invalidation tokens left are intra-session (competing calls on the + * same session), not lifecycle guards. + */ +export class DeployToHubSession { + readonly workspace: string + readonly folder: string + /** `f/`-prefixed folder path the project is scoped to. */ + readonly selectedFolder: string + + #disposed = false + #deps: SessionDeps + + phase = $state('predeploy') + workspaceItems = $state([]) + draftItems = $state([]) + workspaceTriggers = $state([]) + triggersLoading = $state(false) + // True when a trigger kind's discovery failed (not a feature-gated 404): + // the trigger list may be incomplete, so publishing is blocked until a + // retry succeeds. + triggerDiscoveryFailed = $state(false) + schedulePreviews = $state>({}) + manualDeselected = $state>(new Set()) + loading = $state(false) + workspaceRateLimit = $state(undefined) + deploymentStatus = $state< + Record + >({}) + deploying = $state(false) + + recordTarget = $state() + recordArgs = $state>({}) + recordValid = $state(true) + recordSchema = $state>(emptySchema()) + recordSchemaLoading = $state(false) + runState = $state('idle') + runJobId = $state(undefined) + runResult = $state(undefined) + runError = $state(undefined) + recordings = $state>({}) + + // Project-level data-pipeline recording. Unlike script/flow recordings (one + // job per item) a pipeline is the whole folder cascade, so it gets a single + // recording: the resolved asset graph, per-node status timeline, per-node job + // streams and asset samples — replayed by PipelineRecordingReplay. + pipelineGraph = $state(undefined) + pipelineRunState = $state('idle') + pipelineRecordingResult = $state(undefined) + pipelineRunError = $state(undefined) + pipelineRecorded = $state(false) + + publishTarget = $state() + publishing = $state(false) + + hubName = $state('') + hubSummary = $state('') + hubReadme = $state('') + // Custom logo state for the next publish (png/svg, base64 without the + // data: prefix). Three-state: undefined = untouched (publishing leaves the + // Hub's current logo alone), null = clear the Hub's logo on publish, + // object = upload this image. + hubLogo = $state<{ b64: string; mime: string; name: string } | null | undefined>(undefined) + // Whether the Hub currently has a custom logo for this project (from + // rehydration) — drives the "Remove current logo" affordance. + hubHasRemoteLogo = $state(false) + effectiveSlug = $state('') + hubItemIds = $state>({}) + + // Best-effort data table migrations for the bundle, editable in the drawer and + // pushed on deploy. Regenerated when the bundle drawer opens. + migrationDrafts = $state([]) + migrationsGenerating = $state(false) + // Bumped whenever the drafts are (re)generated, to re-key the Monaco editors so + // they pick up the fresh SQL (Monaco doesn't sync external `code` changes). + migrationsGeneration = $state(0) + + bundlePreview = $state(undefined) + detectingResources = $state(false) + // Data tables (→ tables) the current selection reads/writes, detected off the + // same bundle preview. Drives the predeploy "Data table dependencies" summary; + // the editable migration itself is generated in the bundle drawer. + datatableUsage = $state>>(new Map()) + detectingDatatables = $state(false) + + submitting = $state(false) + syncing = $state(false) + + // Intra-session tokens: latest call wins among competing calls on this session. + #triggerLoadTok = 0 + #recordRunTok = 0 + #pipelineRunTok = 0 + #migrationsTok = 0 + #schedulePreviewsInFlight = new Set() + // Preview-only cache: toggling checkboxes re-runs the closure walk, but item + // contents don't change mid-session. deployAll bypasses this and fetches fresh. + #previewItemCache = new Map>() + #previewTypeCache = new Map>() + + constructor(workspace: string, folder: string, deps: SessionDeps) { + this.workspace = workspace + this.folder = folder + this.selectedFolder = `f/${folder}` + this.#deps = deps + } + + dispose() { + this.#disposed = true + // Invalidate any in-flight pipeline cascade poll so it stops on the next + // tick instead of polling to the timeout against a discarded session. + this.#pipelineRunTok++ + } + + load() { + void this.#loadWorkspace() + void this.#loadTriggers() + void this.rehydrateFromHub() + void this.#loadPipelineGraph() + } + + filteredWorkspaceItems = $derived( + this.workspaceItems.filter((i) => i.path.startsWith(this.selectedFolder + '/')) + ) + // Derived (not merged at load time) so it settles regardless of which of the + // racing loads (#loadWorkspace / rehydrateFromHub) finishes last. + draftItemsWithLocalState = $derived(mergeShareState(this.draftItems, this.workspaceItems)) + items = $derived( + this.phase === 'predeploy' ? this.filteredWorkspaceItems : this.draftItemsWithLocalState + ) + selectedItems = $derived( + this.phase === 'predeploy' + ? this.filteredWorkspaceItems.filter((i) => !this.manualDeselected.has(i.key)) + : [] + ) + selectedItemKeys = $derived(this.selectedItems.map((i) => i.key)) + allSelected = $derived( + this.phase === 'predeploy' && + this.selectedItemKeys.length === this.filteredWorkspaceItems.length + ) + recordableItems = $derived(this.items.filter((i) => canRecord(i.kind))) + allRecorded = $derived( + this.recordableItems.length > 0 && this.recordableItems.every((i) => i.rec === 'recorded') + ) + // Pipeline members of this project's folder (`// pipeline` scripts). + pipelineScriptPaths = $derived( + (this.pipelineGraph?.runnables ?? []) + .filter((r) => r.usage_kind === 'script' && r.in_pipeline) + .map((r) => r.path) + ) + // The subset actually in the Hub project — so a member the user deselected from + // the bundle is neither executed nor embedded (with its code/logs/samples) in + // the recording. In the draft phase `items` is the project's membership. + recordablePipelineScriptPaths = $derived( + this.pipelineScriptPaths.filter((p) => + this.items.some((i) => i.kind === 'script' && i.path === p) + ) + ) + pipelineScriptPathSet = $derived(new Set(this.pipelineScriptPaths)) + isPipelineProject = $derived(this.pipelineScriptPaths.length > 0) + hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) + + relevantTriggers = $derived.by(() => { + const selectedScripts = new Set( + this.selectedItems.filter((i) => i.kind === 'script').map((i) => i.path) + ) + const selectedFlows = new Set( + this.selectedItems.filter((i) => i.kind === 'flow').map((i) => i.path) + ) + return this.workspaceTriggers.filter((t) => + t.is_flow ? selectedFlows.has(t.script_path) : selectedScripts.has(t.script_path) + ) + }) + + triggersByKind = $derived.by(() => { + const out = new Map() + for (const t of this.relevantTriggers) { + const arr = out.get(t.kind) ?? [] + arr.push(t) + out.set(t.kind, arr) + } + return Array.from(out.entries()).sort((a, b) => a[0].localeCompare(b[0])) + }) + + runnableSummaryByPath = $derived.by(() => { + const m = new Map() + for (const it of this.workspaceItems) { + if (it.kind === 'script' || it.kind === 'flow') { + m.set(`${it.kind}:${it.path}`, it.summary) + } + } + return m + }) + + // `hasHardcoded` = pinned via $res: path (relocated as a stub); else input-only. + dependencyTypes = $derived.by(() => { + const b = this.bundlePreview + if (!b) return [] as DependencyType[] + const stubByNewPath = new Map(b.resourceStubs.map((s) => [s.newPath, s])) + const byType = new Map() + const ensure = (rt: string) => { + let e = byType.get(rt) + if (!e) { + e = { resource_type: rt, hasHardcoded: false, usages: [] } + byType.set(rt, e) + } + return e + } + for (const it of b.items) { + const label = (it.summary?.trim() || it.path) ?? it.path + const refs = + it.kind === 'flow' + ? extractFlowRefs(it.value).filter((r) => r.kind === 'resource') + : it.kind === 'app' + ? extractAppRefs(it.value) + : extractScriptRefs(it.content ?? '') + for (const r of refs) { + const stub = stubByNewPath.get(r.path) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + const e = ensure(stub.resource_type) + e.hasHardcoded = true + e.usages.push({ + role: 'hardcoded', + label, + kind: it.kind, + path: stub.originalPath, + itemPath: it.path + }) + } + for (const t of typesFromSchema(it.schema)) { + if (HIDDEN_RESOURCE_TYPES.has(t)) continue + ensure(t).usages.push({ role: 'input', label, kind: it.kind, itemPath: it.path }) + } + } + // Resources referenced only by a trigger (no item uses them in code) — + // its kind resource field or any `$res:` token in its config. + const stubByOriginal = new Map(b.resourceStubs.map((s) => [s.originalPath, s])) + for (const t of this.relevantTriggers) { + const refs = new Set( + extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config)) + ) + const rp = triggerResourcePath(t) + if (rp) refs.add(rp) + for (const ref of refs) { + const stub = stubByOriginal.get(ref) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + ensure(stub.resource_type).usages.push({ + role: 'trigger', + label: t.summary?.trim() || t.path, + triggerKind: t.kind, + path: stub.originalPath + }) + } + } + return [...byType.values()].sort((a, b) => a.resource_type.localeCompare(b.resource_type)) + }) + + toggleItem = (item: { key: string }) => { + const next = new Set(this.manualDeselected) + if (next.has(item.key)) next.delete(item.key) + else next.add(item.key) + this.manualDeselected = next + } + selectAll = () => { + this.manualDeselected = new Set() + } + deselectAll = () => { + this.manualDeselected = new Set(this.filteredWorkspaceItems.map((i) => i.key)) + } + + #folderQs(): string { + return `?folder=${encodeURIComponent(this.folder)}` + } + + itemUrl(kind: ItemKind, path: string): string | undefined { + if (!path) return undefined + return `${base}/${ITEM_KIND_ROUTE[kind]}/${path}?workspace=${this.workspace}` + } + triggerListUrl(kind: WorkspaceTriggerKind): string { + return `${base}/${TRIGGER_KINDS[kind].route}?workspace=${this.workspace}` + } + + #patchItem(key: string, patch: Partial) { + this.workspaceItems = this.workspaceItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + this.draftItems = this.draftItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + } + + async #listAllPages( + fetcher: (params: { perPage: number; page: number }) => Promise + ): Promise { + const perPage = 100 + const out: T[] = [] + for (let page = 1; page <= 1000; page++) { + const batch = await fetcher({ perPage, page }) + out.push(...batch) + if (batch.length < perPage) return out + } + return out + } + + async #loadWorkspace() { + const workspace = this.workspace + this.loading = true + try { + const [apps, rawApps, flows, scripts, settings] = await Promise.all([ + this.#listAllPages((p) => AppService.listApps({ workspace, ...p })), + this.#listAllPages((p) => RawAppService.listRawApps({ workspace, ...p })), + this.#listAllPages((p) => FlowService.listFlows({ workspace, ...p })), + this.#listAllPages((p) => ScriptService.listScripts({ workspace, ...p })), + WorkspaceService.getSettings({ workspace }).catch(() => undefined) + ]) + if (this.#disposed) return + + this.workspaceRateLimit = settings?.public_app_execution_limit_per_minute + + const next: DeployItem[] = [] + const publicApps = apps.filter((a) => a.execution_mode === 'anonymous') + const publicUrls = await Promise.all(publicApps.map((a) => this.#resolvePublicUrl(a.path))) + const publicUrlByPath = new Map(publicApps.map((a, i) => [a.path, publicUrls[i]])) + for (const a of apps) { + const isPublic = a.execution_mode === 'anonymous' + // Raw apps live in the `app` table (value = files/runnables) but must be + // published to the Hub as raw apps, not low-code apps. + const isRaw = (a as any).raw_app === true + next.push({ + key: `${isRaw ? 'raw_app' : 'app'}:${a.path}`, + path: a.path, + kind: isRaw ? 'raw_app' : 'app', + appTable: isRaw || undefined, + summary: a.summary, + rec: 'none', + published: isPublic, + publicUrl: isPublic ? publicUrlByPath.get(a.path) : undefined + }) + } + for (const a of rawApps) { + next.push({ + key: `raw_app:${a.path}`, + path: a.path, + kind: 'raw_app', + summary: a.summary, + rec: 'none' + }) + } + for (const f of flows) { + next.push({ + key: `flow:${f.path}`, + path: f.path, + kind: 'flow', + summary: f.summary, + rec: 'none' + }) + } + for (const s of scripts) { + next.push({ + key: `script:${s.path}`, + path: s.path, + kind: 'script', + summary: s.summary, + rec: 'none' + }) + } + if (this.#disposed) return + this.workspaceItems = next + } catch (e: any) { + if (!this.#disposed) { + sendUserToast(`Failed to load project items: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed) this.loading = false + } + } + + /** Re-fetch triggers, e.g. after the EE license hydrates late. */ + reloadTriggers() { + void this.#loadTriggers() + } + + async #loadTriggers() { + const tok = ++this.#triggerLoadTok + this.triggersLoading = true + try { + const { triggers, failedKinds } = await listAllWorkspaceTriggers(this.workspace, { + includeEeOnly: this.#deps.hasEeLicense(), + onError: (message) => { + if (!this.#disposed) sendUserToast(message, true) + } + }) + if (this.#disposed || tok !== this.#triggerLoadTok) return + this.workspaceTriggers = triggers + this.triggerDiscoveryFailed = failedKinds.length > 0 + } finally { + if (!this.#disposed && tok === this.#triggerLoadTok) this.triggersLoading = false + } + } + + async #resolvePublicUrl(path: string): Promise { + try { + const secret = await AppService.getPublicSecretOfApp({ workspace: this.workspace, path }) + return computeSecretUrl(secret) + } catch { + return undefined + } + } + + async rehydrateFromHub() { + try { + const res = await fetch(`/api/w/${this.workspace}/hub/project${this.#folderQs()}`, { + credentials: 'include', + headers: { accept: 'application/json' } + }) + if (this.#disposed) return + if (!res.ok) return // 404 = no project published for this folder yet + const p = JSON.parse(await res.text()) + if (this.#disposed || !p?.slug) return + this.effectiveSlug = p.slug + this.hubName = p.name ?? '' + this.hubSummary = p.summary ?? '' + this.hubReadme = p.readme ?? '' + this.hubHasRemoteLogo = p.has_logo === true + this.phase = + p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' + const ids: Record = {} + this.draftItems = (p.items ?? []).map((it: any) => { + const wpath = it.source_path ?? it.path + const key = `${it.kind}:${wpath}` + if (typeof it.hub_id === 'number') ids[key] = it.hub_id + return { + key, + path: wpath, + kind: it.kind as Kind, + summary: it.summary ?? undefined, + rec: it.has_recording ? 'recorded' : 'none' + } satisfies DeployItem + }) + this.hubItemIds = ids + } catch {} + } + + /** Kick off schedule-preview fetches for any relevant schedule trigger missing one. */ + ensureSchedulePreviews() { + for (const t of this.relevantTriggers) { + if (t.kind !== 'schedule') continue + const c = t.config as any + const key = `${c.schedule}|${c.timezone}` + if (this.schedulePreviews[key] || this.#schedulePreviewsInFlight.has(key)) continue + this.#schedulePreviewsInFlight.add(key) + ScheduleService.previewSchedule({ + requestBody: { + schedule: c.schedule, + timezone: c.timezone, + cron_version: c.cron_version ?? 'v2' + } + }) + .then((dates) => { + this.schedulePreviews = { ...this.schedulePreviews, [key]: dates.slice(0, 3) } + }) + .catch(() => {}) + .finally(() => this.#schedulePreviewsInFlight.delete(key)) + } + } + + /** + * Rebuild the predeploy bundle preview (resource + data table dependency + * summaries), debounced so rapid checkbox toggles coalesce into one walk. + * Reads its reactive inputs synchronously and returns a cancel function, so + * it can be driven from an `$effect` with proper cleanup. + */ + queueBundlePreview(): (() => void) | undefined { + if (this.phase !== 'predeploy') { + this.bundlePreview = undefined + this.datatableUsage = new Map() + return undefined + } + this.detectingResources = true + this.detectingDatatables = true + const slug = this.hubSlug + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, slug) + ] + const triggerResources = this.#triggerResourcePaths(this.relevantTriggers) + const triggerVars = this.#triggerVarPaths(this.relevantTriggers) + let cancelled = false + const timer = setTimeout(() => { + buildProjectBundle(seed, slug, this.#cachedBundleDeps(), triggerResources, triggerVars) + .then((b) => { + if (cancelled) return + this.bundlePreview = b + // Detect data table usage off the same fetched items. + detectDatatableTables(b.items) + .then((usage) => { + if (!cancelled) this.datatableUsage = usage + }) + .finally(() => { + if (!cancelled) this.detectingDatatables = false + }) + }) + .finally(() => { + if (!cancelled) this.detectingResources = false + }) + }, 250) + return () => { + cancelled = true + clearTimeout(timer) + } + } + + #buildBundleDeps(): BundleDeps { + const workspace = this.workspace + return { + fetchItem: async (ref: ItemRef): Promise => { + try { + if (ref.kind === 'script') { + const s = await ScriptService.getScriptByPath({ workspace, path: ref.path }) + return { + kind: 'script', + path: ref.path, + summary: s.summary, + description: s.description ?? undefined, + content: s.content, + language: s.language, + schema: s.schema, + lock: s.lock ?? undefined, + scriptKind: typeof s.kind === 'string' ? s.kind.toLowerCase() : 'script' + } + } else if (ref.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace, path: ref.path }) + return { + kind: 'flow', + path: ref.path, + summary: f.summary, + description: f.description ?? undefined, + value: f.value, + schema: f.schema + } + } else if (ref.kind === 'app') { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + return { kind: 'app', path: ref.path, summary: a.summary, value: a.value } + } else if (ref.kind === 'raw_app') { + // Modern raw apps live in the `app` table: fetch source files + + // runnables + the compiled bundle, and shape them into the `raw` + // payload the Hub's RawAppView expects (JSON is valid YAML). + const isModern = this.workspaceItems.some( + (i) => i.kind === 'raw_app' && i.path === ref.path && i.appTable + ) + if (isModern) { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ + workspace, + path: ref.path + }) + // The compiled JS bundle is required; a missing one means the app + // was never built/deployed, so fail loudly instead of pushing a blank app. + const [jsRes, cssRes] = await Promise.all([ + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.js`, { + credentials: 'include' + }), + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.css`, { + credentials: 'include' + }) + ]) + if (!jsRes.ok) { + throw new Error(`raw app ${ref.path} has no compiled bundle — deploy it first`) + } + const js = await jsRes.text() + const css = cssRes.ok ? await cssRes.text() : '' + const v: any = a.value ?? {} + const content = JSON.stringify({ + files: { ...(v.files ?? {}), '/bundle.js': js, '/bundle.css': css }, + runnables: v.runnables ?? {}, + // Preserve the full-code app's explicit data table declaration so it + // survives publish/import and feeds migration detection. + ...(v.data !== undefined ? { data: v.data } : {}), + ...(v.datatables !== undefined ? { datatables: v.datatables } : {}) + }) + return { kind: 'raw_app', path: ref.path, summary: a.summary, content } + } + const r = await fetch(`/api/w/${workspace}/raw_apps/get_data/0/${ref.path}`, { + credentials: 'include' + }) + if (!r.ok) return undefined + return { kind: 'raw_app', path: ref.path, content: await r.text() } + } + } catch (e: any) { + return undefined + } + return undefined + }, + resolveResourceType: async (path: string): Promise => { + try { + const r = await ResourceService.getResource({ workspace, path }) + return r.resource_type ?? undefined + } catch (e: any) { + return undefined + } + } + } + } + + #cachedBundleDeps(): BundleDeps { + const deps = this.#buildBundleDeps() + // Memoize only successful lookups: a miss (undefined) is likely transient, so + // evict it once it resolves. Otherwise a fixed/retried dependency can never + // clear `bundlePreview.unresolved` until the whole session is recreated. + const memoize = ( + cache: Map>, + key: string, + run: () => Promise + ) => { + let p = cache.get(key) + if (!p) { + p = run() + cache.set(key, p) + void p.then((r) => { + if (r === undefined && cache.get(key) === p) cache.delete(key) + }) + } + return p + } + return { + fetchItem: (ref) => + memoize(this.#previewItemCache, `${ref.kind}:${ref.path}`, () => deps.fetchItem(ref)), + resolveResourceType: (path) => + memoize(this.#previewTypeCache, path, () => deps.resolveResourceType(path)) + } + } + + async #postHub(path: string, body: unknown): Promise | undefined> { + const res = await fetch(`/api/w/${this.workspace}${path}${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(body) + }) + const text = await res.text() + if (!res.ok) throw new Error(text) + try { + return JSON.parse(text) + } catch { + return undefined + } + } + + async regenerateMigrations() { + const tok = ++this.#migrationsTok + this.migrationsGenerating = true + try { + // Same handler-augmented seed as deployAll: a data table used only by a + // bundled trigger handler must still get its migration. + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, this.hubSlug || 'project') + ] + // Detection is independent of the final slug (data table refs aren't + // relocated), so any placeholder slug works for this throwaway bundle. + const bundle = await buildProjectBundle( + seed, + this.hubSlug || 'project', + this.#buildBundleDeps(), + [] + ) + const usage = await detectDatatableTables(bundle.items) + const drafts = await generateDatatableMigrations(this.workspace, usage) + if (this.#disposed || tok !== this.#migrationsTok) return + this.migrationDrafts = drafts + this.migrationsGeneration++ + } catch (e: any) { + if (!this.#disposed && tok === this.#migrationsTok) { + this.migrationDrafts = [] + this.migrationsGeneration++ + // Toast so a genuine failure isn't mistaken for "no data table usage". + sendUserToast(`Could not generate data table migrations: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed && tok === this.#migrationsTok) this.migrationsGenerating = false + } + } + + /** Prefill bundle metadata and start migration detection (bundle drawer opening). */ + prepareBundle() { + this.hubName = this.hubName || this.folder + void this.regenerateMigrations() + } + + /** + * Create the Hub draft then push the full bundle. `deploying` is set + * synchronously before the first request so a double-click cannot start a + * second publish, and the whole run is refused while triggers are still + * loading — an incomplete `relevantTriggers` snapshot would permanently + * omit triggers (and their handlers and migrations) from the draft. + * `onDraftCreated` fires once the draft exists (the bundle drawer closes + * there while items continue publishing). + */ + async publishBundle(onDraftCreated?: () => void): Promise { + if (this.deploying || this.triggersLoading || this.triggerDiscoveryFailed) return + this.deploying = true + try { + if (!(await this.#createDraft())) return + onDraftCreated?.() + await this.#deployAll() + } finally { + this.deploying = false + } + } + + /** + * Create the Hub draft project. Returns true when the draft exists and + * publishing can proceed. + */ + async #createDraft(): Promise { + this.hubName = this.hubName.trim() + this.hubSummary = this.hubSummary.trim() + this.hubReadme = this.hubReadme.trim() + try { + const res = await fetch(`/api/w/${this.workspace}/hub/publish_draft${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + slug: this.hubSlug, + name: this.hubName, + summary: this.hubSummary || this.hubName, + readme: this.hubReadme || undefined + }) + }) + const text = await res.text() + if (!res.ok) { + sendUserToast(`Hub draft creation failed: ${text}`, true) + return false + } + // Abort if Hub didn't echo a slug — guessing here lands items under + // a folder the Hub never locked. + let returnedSlug: string | undefined + try { + const parsed = JSON.parse(text) + if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + } catch {} + if (!returnedSlug) { + sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) + return false + } + // Session replaced mid-request (workspace/folder switch): publishing now + // would push another scope's items into this draft. Abort. + if (this.#disposed) { + sendUserToast(`Workspace changed during publish — aborted to avoid mixing items.`, true) + return false + } + this.effectiveSlug = returnedSlug + return true + } catch (e: any) { + sendUserToast(`Hub draft creation failed: ${e?.message ?? e}`, true) + return false + } + } + + async #pushBundledItem(slug: string, it: BundledItem): Promise { + const key = `${it.kind}:${it.path}` + if (it.kind === 'script') { + const resp = await this.#postHub('/hub/scripts', { + summary: it.summary || it.newPath, + app: slug, + description: it.description ?? '', + kind: it.scriptKind ?? 'script', + content: it.content, + language: it.language, + schema: it.schema ?? undefined, + lockfile: it.lock ?? undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'flow') { + const resp = await this.#postHub('/hub/flows', { + flow: { + summary: it.summary || it.newPath, + description: it.description ?? undefined, + value: it.value, + schema: it.schema ?? undefined + }, + apps: [], + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'app') { + await this.#postHub('/hub/apps', { + app: it.value, + apps: [], + summary: it.summary || it.newPath, + description: undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + } else if (it.kind === 'raw_app') { + const resp = await this.#postHub('/hub/raw_apps', { + raw: it.content ?? '', + apps: [], + summary: it.summary || it.newPath, + path: it.newPath, + source_path: it.path, + description: undefined, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } + } + + // Handler runnables (trigger error handlers, schedule on_* handlers) ship + // with the bundle like the primary runnables do; hub refs stay external. + #triggerHandlerSeed(triggers: WorkspaceTrigger[], slug: string): ItemRef[] { + return triggers.flatMap(triggerHandlerRefs).filter((r) => classifyPath(r.path, slug) !== 'hub') + } + + // Every resource a trigger's exported config references: the kind-specific + // broker/auth field plus any `$res:` token nested in it (schedule args, + // handler extra args, …) — all must enter the bundle path map. + #triggerResourcePaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + const rp = triggerResourcePath(t) + if (rp) out.add(rp) + for (const p of extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config))) { + out.add(p) + } + } + return [...out] + } + + // Every whole-string `$var:`/`$jsonvar:` value a trigger's config resolves (SQS + // queue_url, schedule args, …) — relocated through the bundle map like item vars. + #triggerVarPaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + for (const p of extractVarRefsFromValue(portableTriggerConfig(t.kind, t.config))) out.add(p) + } + return [...out] + } + + async #pushTriggers( + slug: string, + resourcePathMap: Map, + relevant: WorkspaceTrigger[] + ): Promise { + const pathMap = buildPathMap( + relevant.map((t) => t.path), + slug + ) + const triggers: Array> = [] + const skipped: string[] = [] + for (const t of relevant) { + const itemKind: ItemKind = t.is_flow ? 'flow' : 'script' + const runnableKey = `${itemKind}:${t.script_path}` + const hubId = this.hubItemIds[runnableKey] + if (!hubId) { + skipped.push(t.path) + continue + } + // Full-config remap: resource paths, error-handler paths, schedule on_* + // handler refs and whole-string `$var:` values all relocate through the map. + const config = rewriteVarRefsInValue( + rewriteTriggerConfig(portableTriggerConfig(t.kind, t.config), resourcePathMap), + resourcePathMap + ) + triggers.push({ + path: pathMap.get(t.path) ?? t.path, + kind: t.kind, + summary: t.summary ?? null, + description: (t.config as any)?.description ?? null, + config, + script_ask_id: t.is_flow ? null : hubId, + flow_id: t.is_flow ? hubId : null + }) + } + if (skipped.length > 0) { + sendUserToast( + `Skipped ${skipped.length} trigger(s) whose runnable did not publish: ${skipped.join(', ')}`, + true + ) + } + // Full-set sync: always push (an empty list clears the Hub's triggers on a + // re-deploy), so removing every trigger doesn't leave stale ones on the Hub. + await this.#postHub('/hub/triggers', { triggers, project_slug: slug }) + } + + // Builtin types (git_repository, ...) aren't in resource_type — push with empty schema. + async #pushResourceTypes(slug: string, types: string[]): Promise { + const results = await Promise.all( + types.map(async (name) => { + let schema: unknown = undefined + let description: string | undefined = undefined + try { + const rt = await ResourceService.getResourceType({ + workspace: this.workspace, + path: name + }) + schema = rt.schema ?? undefined + description = rt.description ?? undefined + } catch (e: any) {} + try { + await this.#postHub('/hub/resource_types', { + name, + schema, + description, + project_slug: slug + }) + return 0 + } catch (e: any) { + sendUserToast(`Resource type ${name} push failed: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + return results.reduce((a: number, b) => a + b, 0) + } + + async #deployAll() { + const slug = this.hubSlug + // Snapshot the selection up-front: `selectedItems`/`relevantTriggers` are + // derived from live workspace data and `migrationDrafts` is edited in the + // drawer — the deploy must publish exactly what the user confirmed. + const itemsSnapshot = this.selectedItems.slice() + const triggersSnapshot = this.relevantTriggers.slice() + const migrationsSnapshot = this.migrationDrafts.slice() + this.hubItemIds = {} + this.deploymentStatus = {} + let failures = 0 + try { + const seed: ItemRef[] = [ + ...itemsSnapshot + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(triggersSnapshot, slug) + ] + const triggerResources = this.#triggerResourcePaths(triggersSnapshot) + const triggerVars = this.#triggerVarPaths(triggersSnapshot) + const bundle = await buildProjectBundle( + seed, + slug, + this.#buildBundleDeps(), + triggerResources, + triggerVars + ) + // Full path map (incl. unresolved) so a trigger's resource path is always + // relocated — never leaks the publisher's original private path to the Hub. + const resourcePathMap = bundle.pathMap + + // A dangling reference (a selected root or transitive runnable that failed + // to fetch, or a resource whose type can't be resolved) means the bundle + // doesn't close: the root would silently vanish, or a published item would + // still point at the publisher's private source-workspace path. Refuse to + // publish until every reference resolves rather than ship a broken project. + if (bundle.unresolved.length > 0) { + sendUserToast( + `Cannot publish: ${bundle.unresolved.length} unresolved reference(s): ${bundle.unresolved.join(', ')}. Deselect or fix them, then retry.`, + true + ) + return + } + + // Bundle building is slow — bail before the first Hub write if the session + // was replaced (workspace/folder switch) in the meantime. + if (this.#disposed) return + + // Types come from $res: stubs AND schema inputs (resource-). + const inputTypes = bundle.items + .flatMap((i) => typesFromSchema(i.schema)) + .filter((t) => !HIDDEN_RESOURCE_TYPES.has(t)) + const types = [ + ...new Set([...bundle.resourceStubs.map((s) => s.resource_type), ...inputTypes]) + ] + const depFailures = await this.#pushResourceTypes(slug, types) + + // Input-type deps with no path get a conventional f// stub. + const stubsByPath = new Map() + for (const s of bundle.resourceStubs) + stubsByPath.set(s.newPath, { path: s.newPath, resource_type: s.resource_type }) + for (const t of inputTypes) { + const path = `f/${slug}/${t}` + if (!stubsByPath.has(path)) stubsByPath.set(path, { path, resource_type: t }) + } + const stubs = [...stubsByPath.values()] + if (stubs.length > 0) { + try { + await this.#postHub('/hub/resources', { resources: stubs, project_slug: slug }) + } catch (e: any) { + sendUserToast(`Resource sync failed: ${e?.message ?? e}`, true) + failures++ + } + } + failures += depFailures + if (failures > 0) { + sendUserToast( + `Resource dependency sync failed — items not published to avoid broken references.`, + true + ) + return + } + + for (const it of bundle.items) { + // Stop writing item status / Hub IDs once the session is replaced — + // continuing would publish into a project the user has moved away from. + if (this.#disposed) return + const key = `${it.kind}:${it.path}` + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'loading' } } + try { + await this.#pushBundledItem(slug, it) + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'deployed' } } + } catch (e: any) { + failures++ + this.deploymentStatus = { + ...this.deploymentStatus, + [key]: { status: 'failed', error: e?.message ?? String(e) } + } + } + } + // A re-bundle clears the Hub-side embed (idempotent replace), so re-push it + // for any raw app that is already public — keeps the live iframe in sync + // without forcing an unpublish/share round-trip. Updates by hub id, safe in parallel. + const embedResults = await Promise.all( + bundle.items + .filter((it) => it.kind === 'raw_app') + .map(async (it) => { + const hubId = this.hubItemIds[`${it.kind}:${it.path}`] + const src = itemsSnapshot.find((i) => i.kind === 'raw_app' && i.path === it.path) + if (!hubId || !src?.published) return 0 + // The re-bundle cleared the embed; a public raw app with no resolved URL + // can't have its iframe restored, so it's an incomplete publish too — + // count it (like a push failure) so the draft can't become submit-ready. + if (!src.publicUrl) { + sendUserToast(`Cannot restore the iframe for ${it.path}: missing public URL`, true) + return 1 + } + try { + await this.#pushRawAppEmbed(hubId, src.publicUrl) + return 0 + } catch (e: any) { + sendUserToast(`Failed to sync iframe for ${it.path}: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + failures += embedResults.reduce((a: number, b) => a + b, 0) + if (this.#disposed) return + try { + await this.#pushTriggers(slug, resourcePathMap, triggersSnapshot) + } catch (e: any) { + sendUserToast(`Trigger sync failed: ${e?.message ?? e}`, true) + failures++ + } + + // Full-set sync: always push (an empty list clears the Hub's migrations on + // a re-deploy). The Hub drops empty-SQL entries, so disabled placeholders + // don't persist. + try { + await this.#postHub('/hub/migrations', { + migrations: migrationsSnapshot.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down, + enabled: m.enabled + })), + project_slug: slug + }) + } catch (e: any) { + sendUserToast(`Data table migration sync failed: ${e?.message ?? e}`, true) + failures++ + } + + // Push the logo only when touched this session: an object uploads it, + // null clears the Hub's current logo, undefined leaves it alone + // (re-publishing a bundle must not clear it). + if (this.hubLogo !== undefined) { + try { + await this.#postHub(`/hub/projects/${encodeURIComponent(slug)}/logo`, { + logo: this.hubLogo ? { b64: this.hubLogo.b64, mime: this.hubLogo.mime } : null + }) + this.hubHasRemoteLogo = this.hubLogo !== null + this.hubLogo = undefined + } catch (e: any) { + sendUserToast(`Logo ${this.hubLogo ? 'upload' : 'removal'} failed: ${e?.message ?? e}`, true) + failures++ + } + } + + await sleep(150) + if (this.#disposed) return + // An incomplete push must never become submittable: a failed transitive item + // can leave a pushed runnable pointing at content that never landed. Stay in + // predeploy (deploymentStatus keeps the failed items visible) so re-publishing + // retries every write — createDraft and the item pushes are idempotent. + if (failures > 0) { + sendUserToast( + `Publish incomplete: ${failures} write(s) failed. Nothing was submitted — fix them and re-publish.`, + true + ) + return + } + this.deploymentStatus = {} + this.recordings = {} + // Deterministic baseline so a transient Hub read failure can't leave the + // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. + this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + } finally { + this.deploying = false + } + } + + submitForReview = async () => { + const slug = this.hubSlug + if (!slug) return + this.submitting = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/submit${this.#folderQs()}`, + { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + } + ) + if (!res.ok) { + sendUserToast(`Submit for review failed: ${await res.text()}`, true) + return + } + this.phase = 'under_review' + sendUserToast('Submitted for review by the Windmill team.') + } finally { + this.submitting = false + } + } + + syncWithHub = async () => { + this.syncing = true + try { + if (this.phase === 'draft') { + await this.#loadWorkspace() + const prev = new Map(this.draftItems.map((i) => [i.key, { rec: i.rec }])) + this.draftItems = this.workspaceItems + .filter((i) => prev.has(i.key)) + .map((i) => ({ ...i, rec: prev.get(i.key)?.rec ?? 'none' })) + } else { + // under_review / live: re-fetch the Hub project to pick up an + // admin status change (under_review -> live). + const before = this.phase + await this.rehydrateFromHub() + sendUserToast( + this.phase === before + ? 'Still waiting for review.' + : this.phase === 'live' + ? 'Approved — your project is now live.' + : `Status updated: ${this.phase}.` + ) + } + } catch (e: any) { + sendUserToast(`Sync failed: ${e?.message ?? e}`, true) + } finally { + this.syncing = false + } + } + + startNewDraft = () => { + this.draftItems = [] + this.recordings = {} + this.phase = 'predeploy' + } + + /** Reset record-drawer state and load the target's schema. */ + async openRecord(it: DeployItem) { + const tok = ++this.#recordRunTok + this.recordTarget = it + this.recordArgs = {} + this.recordValid = true + this.recordSchema = emptySchema() + this.recordSchemaLoading = true + this.runState = 'idle' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + if (it.kind === 'script') { + const s = await ScriptService.getScriptByPath({ + workspace: this.workspace, + path: it.path + }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (s.schema as Record) ?? emptySchema() + } else if (it.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace: this.workspace, path: it.path }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (f.schema as Record) ?? emptySchema() + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + sendUserToast(`Failed to load schema: ${e?.message ?? e}`, true) + } finally { + if (tok === this.#recordRunTok) this.recordSchemaLoading = false + } + } + + /** Invalidate any in-flight record run/poll (record drawer closed). */ + cancelRecordRun = () => { + this.#recordRunTok++ + } + + runJob = async () => { + const it = this.recordTarget + if (!it) return + const tok = ++this.#recordRunTok + this.runState = 'running' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + let jobId: string + if (it.kind === 'script') { + jobId = await JobService.runScriptByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else if (it.kind === 'flow') { + jobId = await JobService.runFlowByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else { + if (tok === this.#recordRunTok) this.runState = 'idle' + return + } + if (tok !== this.#recordRunTok) return + this.runJobId = jobId + await this.#pollJobUntilComplete(jobId, tok) + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Failed to start: ${e?.message ?? e}` + } + } + + async #pollJobUntilComplete(jobId: string, tok: number) { + // First check immediately (fast scripts complete in ms), then back off to 2s. + const deadline = Date.now() + 5 * 60_000 + let interval = 250 + while (Date.now() < deadline) { + if (tok !== this.#recordRunTok) return + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId + }) + if (tok !== this.#recordRunTok) return + if (r.completed) { + this.runResult = r.result + if (r.success) { + this.runState = 'success' + } else { + this.runState = 'failed' + this.runError = typeof r.result === 'string' ? r.result : JSON.stringify(r.result) + } + return + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Polling failed: ${e?.message ?? e}` + return + } + await sleep(interval) + interval = Math.min(interval * 2, 2000) + } + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = 'Timed out after 5 minutes' + } + + async #buildScriptRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const s = await ScriptService.getScriptByPath({ workspace, path: it.path }) + const job = await JobService.getCompletedJob({ workspace, id: jobId }) + const initial_job = { ...(job as any), type: 'CompletedJob' } + const events = [{ t: 0, data: { completed: true, job: initial_job } }] + const duration = (initial_job.duration_ms as number) ?? 0 + return { + version: 1, + type: 'script' as const, + recorded_at: new Date().toISOString(), + script_path: it.path, + total_duration_ms: duration, + code: s.content, + language: s.language, + args: (job.args ?? {}) as Record, + schema: s.schema, + job: { initial_job, events } + } + } + + async #buildFlowRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const f = await FlowService.getFlowByPath({ workspace, path: it.path }) + const root = (await JobService.getCompletedJob({ workspace, id: jobId })) as any + const jobs: Record = {} + const collect = async (j: any) => { + const stamped = { ...j, type: 'CompletedJob' } + jobs[j.id] = { + initial_job: stamped, + events: [{ t: 0, data: { completed: true, job: stamped } }] + } + const modules = (j.flow_status?.modules ?? []).filter( + (m: any) => m.job && typeof m.job === 'string' + ) + // Sub-jobs at the same level are independent reads. + await Promise.all( + modules.map(async (m: any) => { + try { + const sub = (await JobService.getCompletedJob({ workspace, id: m.job })) as any + await collect(sub) + } catch { + /* sub-job missing — skip */ + } + }) + ) + } + await collect(root) + return { + version: 1, + recorded_at: new Date().toISOString(), + flow_path: it.path, + total_duration_ms: (root.duration_ms as number) ?? 0, + flow: { + path: it.path, + value: f.value, + schema: f.schema ?? { type: 'object', properties: {}, required: [] }, + summary: f.summary ?? '', + archived: false, + edited_at: '', + edited_by: '', + extra_perms: {} + }, + jobs + } + } + + /** Save the current successful run as the Hub recording. Returns true on success. */ + async saveRecording(): Promise { + const it = this.recordTarget + if (!it || !this.runJobId || this.runState !== 'success') return false + const hubId = this.hubItemIds[it.key] + if (!hubId) { + sendUserToast(`Push the bundle to the Hub first before saving recordings`, true) + return false + } + if (it.kind !== 'script' && it.kind !== 'flow') { + sendUserToast(`Recordings only supported for script/flow`, true) + return false + } + try { + const recording = + it.kind === 'script' + ? await this.#buildScriptRecording(it, this.runJobId) + : await this.#buildFlowRecording(it, this.runJobId) + const path = it.kind === 'script' ? 'scripts' : 'flows' + await this.#postHub(`/hub/${path}/${hubId}/recording`, { + recording, + project_slug: this.hubSlug + }) + this.recordings = { ...this.recordings, [it.key]: this.runJobId } + this.#patchItem(it.key, { rec: 'recorded' }) + sendUserToast(`Recording saved — job ${this.runJobId}`) + return true + } catch (e: any) { + sendUserToast(`Failed to save recording: ${e?.message ?? e}`, true) + return false + } + } + + /** Resolve the project folder's asset graph so a data-pipeline project can be + * detected and its whole-folder cascade recorded. Best-effort — a project + * with no pipeline just never shows the pipeline record card. */ + async #loadPipelineGraph() { + try { + const params = new URLSearchParams({ + folder: this.folder, + asset_kinds: DATA_ASSET_KINDS.join(',') + }) + const res = await fetch(`/api/w/${this.workspace}/assets/graph?${params}`, { + credentials: 'include' + }) + if (!res.ok) throw new Error(`GET /assets/graph → ${res.status}`) + const graph = (await res.json()) as AssetGraphResponse + if (this.#disposed) return + this.pipelineGraph = graph + } catch { + // No pipeline graph — the pipeline record card simply stays hidden. + } + } + + /** Run the whole-folder cascade and capture it into a single PipelineRecording. + * Deployed-only (no drafts) and arg-less — unlike the editor it seeds no + * per-node input, so a root that needs uploaded data or a schedule's static + * payload records a failure the user can see and fix rather than a green run. */ + runPipelineRecording = async () => { + const fullGraph = this.pipelineGraph + const scripts = this.recordablePipelineScriptPaths + if (!fullGraph || scripts.length === 0) return + const scriptSet = new Set(scripts) + // Scope the graph to the project's members so the run, the recorded graph + // (rendered by the player) and the asset samples all exclude deselected + // branches. + const graph = pruneGraphToScripts(fullGraph, scriptSet) + const tok = ++this.#pipelineRunTok + this.pipelineRunState = 'running' + this.pipelineRecordingResult = undefined + this.pipelineRunError = undefined + // A previous save's badge must not linger over a fresh, unsaved re-run. + this.pipelineRecorded = false + const workspace = this.workspace + try { + const { recording, result } = await capturePipelineRecording({ + workspace, + folder: this.folder, + graph, + scriptPaths: scriptSet, + launch: (path) => + JobService.runScriptByPath({ + workspace, + path, + // Skip the backend asset-trigger dispatcher: the cascade engine owns + // the whole closure (parity with the pipeline editor's bounded run). + requestBody: { _wmill_skip_asset_dispatch: true } + }), + waitTerminal: (jobId) => this.#waitJobTerminal(jobId, tok) + }) + if (tok !== this.#pipelineRunTok) return + this.pipelineRecordingResult = recording + // A dependency cycle drops its members from the schedule, so an all- or + // partially-cyclic run leaves the recording missing steps (and an empty + // schedule reports `ok`). Treat any dropped cyclic member as a failure so + // an incomplete pipeline can't be saved as a successful recording. + if (result.cyclic.length > 0) { + this.pipelineRunState = 'failed' + this.pipelineRunError = `Cannot record — ${result.cyclic.length} script(s) on a dependency cycle: ${result.cyclic.join(', ')}` + } else if (result.ok) { + this.pipelineRunState = 'success' + } else { + this.pipelineRunState = 'failed' + const failed = [...result.statuses.entries()] + .filter(([, s]) => s.status === 'failure') + .map(([p]) => p) + this.pipelineRunError = + failed.length > 0 ? `Failed at ${failed.join(', ')}` : 'Cascade did not complete' + } + } catch (e: any) { + if (tok !== this.#pipelineRunTok) return + this.pipelineRunState = 'failed' + this.pipelineRunError = `Failed to run pipeline: ${e?.message ?? e}` + } + } + + // Poll a launched step to terminal, matching the pipeline editor's cascade + // timeout (DuckLake/DuckDB steps routinely exceed a few minutes). Adds the + // `#pipelineRunTok` cancellation the shared `makeWaitJobTerminal` lacks. + async #waitJobTerminal(jobId: string, tok: number): Promise<'success' | 'failure'> { + const deadline = Date.now() + CASCADE_JOB_TIMEOUT_MS + while (Date.now() < deadline) { + if (tok !== this.#pipelineRunTok) throw new Error('cancelled') + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId, + getStarted: false + }) + if (r.completed) return r.success ? 'success' : 'failure' + } catch { + // transient — retry on the next tick + } + await sleep(CASCADE_POLL_INTERVAL_MS) + } + throw new Error(`Timed out waiting for job ${jobId}`) + } + + /** Save the captured pipeline recording to the Hub, scoped to the project + * (a pipeline is the whole folder, not a single Hub item). Returns true on + * success. */ + async savePipelineRecording(): Promise { + const recording = this.pipelineRecordingResult + if (!recording || this.pipelineRunState !== 'success') return false + if (this.phase === 'predeploy') { + sendUserToast(`Push the project to the Hub first before saving its pipeline recording`, true) + return false + } + try { + await this.#postHub(`/hub/projects/${this.hubSlug}/pipeline_recording`, { recording }) + this.pipelineRecorded = true + sendUserToast(`Pipeline recording saved`) + return true + } catch (e: any) { + sendUserToast(`Failed to save pipeline recording: ${e?.message ?? e}`, true) + return false + } + } + + // Set the Hub raw app's live-iframe URL (or clear it with null). The Hub renders + // from external_embed_url; project_slug scopes ownership. + async #pushRawAppEmbed(hubId: number, url: string | null) { + await this.#postHub(`/hub/raw_apps/${hubId}/embed`, { + external_embed_url: url, + project_slug: this.hubSlug + }) + } + + // Flip an app/raw app between public (anonymous) and private (publisher) and keep + // the Hub raw-app iframe in sync. Returns the resolved public URL when shared. + async #setAppShared(it: DeployItem, shared: boolean): Promise { + const workspace = this.workspace + const hubId = it.kind === 'raw_app' ? this.hubItemIds[it.key] : undefined + // Sharing a raw app as an iframe needs its Hub item to wire the embed. Fail + // before flipping the app public so it can't be left anonymous with no embed. + if (shared && it.kind === 'raw_app' && !hubId) { + throw new Error('Push the bundle to the Hub first to share the live iframe') + } + const app = await AppService.getAppByPath({ workspace, path: it.path }) + const prevMode = (app.policy?.execution_mode ?? 'publisher') as 'anonymous' | 'publisher' + const nextMode = (shared ? 'anonymous' : 'publisher') as 'anonymous' | 'publisher' + const setMode = (mode: 'anonymous' | 'publisher', message: string) => + AppService.updateApp({ + workspace, + path: it.path, + requestBody: { + policy: { ...(app.policy ?? {}), execution_mode: mode }, + deployment_message: message + } + }) + // Undo the policy flip so the app's public state stays consistent when a later + // step of the share fails. Best-effort: a revert failure must not mask the cause. + const rollback = () => setMode(prevMode, 'Revert iframe share').catch(() => {}) + await setMode(nextMode, shared ? 'Share as iframe' : 'Unshare iframe') + const url = shared ? ((await this.#resolvePublicUrl(it.path)) ?? null) : null + // A share with no resolvable public URL is incomplete (no embeddable link, no + // Unpublish control); don't leave the app anonymous while reporting success. + if (shared && url === null) { + await rollback() + throw new Error(`Could not resolve the public URL for ${it.path}`) + } + if (hubId && it.kind === 'raw_app' && (!shared || url)) { + try { + await this.#pushRawAppEmbed(hubId, shared ? url : null) + } catch (e) { + await rollback() + throw e + } + } + return url + } + + /** Make the publish target public. Returns true on success. */ + async confirmPublish(): Promise { + const it = this.publishTarget + if (!it || !canShareAsIframe(it)) return false + this.publishing = true + try { + const url = await this.#setAppShared(it, true) + this.#patchItem(it.key, { published: true, publicUrl: url ?? undefined }) + sendUserToast(`${it.path} is now public`) + return true + } catch (e: any) { + sendUserToast(`Failed to publish: ${e?.message ?? e}`, true) + return false + } finally { + this.publishing = false + } + } + + unpublishApp = async (it: DeployItem) => { + if (!canShareAsIframe(it)) return + try { + await this.#setAppShared(it, false) + this.#patchItem(it.key, { published: false, publicUrl: undefined }) + sendUserToast('App unpublished') + } catch (e: any) { + sendUserToast(`Failed to unpublish: ${e?.message ?? e}`, true) + } + } +} + +/** + * Owns the session lifecycle: a new `DeployToHubSession` is created whenever the + * (workspace, folder) identity actually changes — a spurious same-value store + * emit reuses the live session — and the previous one is disposed, which is the + * single mechanism invalidating its in-flight work. Also hosts the reactive + * plumbing the session itself can't (license-hydration reload, schedule + * previews, debounced bundle preview). + */ +export function useDeployToHubSession(args: { + workspace: () => string | undefined + folder: () => string + hasEeLicense: () => boolean +}) { + let session = $state() + + $effect(() => { + const workspace = args.workspace() + const folder = args.folder() + if (!workspace) return + untrack(() => { + if (session && session.workspace === workspace && session.folder === folder) return + session?.dispose() + const next = new DeployToHubSession(workspace, folder, { + hasEeLicense: args.hasEeLicense + }) + session = next + next.load() + }) + }) + + // The EE license hydrates async; if it lands after a license-less trigger load, + // EE kinds stay empty. Re-fetch on false→true (the session reads the license + // getter at call time). + let prevHadLicense: boolean | undefined = undefined + $effect(() => { + const hasLicense = args.hasEeLicense() + untrack(() => { + if (hasLicense && prevHadLicense === false) session?.reloadTriggers() + prevHadLicense = hasLicense + }) + }) + + // Leaving/entering predeploy invalidates manual selection tweaks. + $effect(() => { + const s = session + if (!s) return + s.phase + untrack(() => { + s.manualDeselected = new Set() + }) + }) + + // Schedule previews for relevant schedule triggers (deduped in the session). + $effect(() => { + session?.ensureSchedulePreviews() + }) + + // Debounced predeploy bundle preview; the session reads its reactive inputs + // synchronously and returns the cancel function used as effect cleanup. + $effect(() => { + const s = session + if (!s) return + return s.queueBundlePreview() + }) + + return { + get session() { + return session + } + } +} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts new file mode 100644 index 0000000000..7b61251fcc --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest' +import { canShareAsIframe, mergeShareState, type DeployItem } from './deployToHubSession.svelte' + +function item(over: Partial & Pick): DeployItem { + return { rec: 'none', ...over } +} + +describe('canShareAsIframe', () => { + it('allows low-code apps and app-table raw apps', () => { + expect(canShareAsIframe(item({ key: 'app:f/a', path: 'f/a', kind: 'app' }))).toBe(true) + expect( + canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })) + ).toBe(true) + }) + it('hides the action for legacy raw apps (raw_app table only)', () => { + // Legacy entries from RawAppService carry no appTable flag; AppService can't load them. + expect(canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' }))).toBe(false) + }) + it('never offers the action for flows or scripts', () => { + expect(canShareAsIframe(item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' }))).toBe(false) + }) +}) + +describe('mergeShareState', () => { + it('carries live public-share state from workspace items onto matching drafts', () => { + const drafts = [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })] + const workspace = [ + item({ + key: 'app:f/a', + path: 'f/a', + kind: 'app', + published: true, + publicUrl: 'https://x/app' + }) + ] + const merged = mergeShareState(drafts, workspace) + expect(merged[0].published).toBe(true) + expect(merged[0].publicUrl).toBe('https://x/app') + }) + it('restores the app-table origin so app-table raw apps stay shareable', () => { + const drafts = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' })] + const workspace = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })] + expect(canShareAsIframe(mergeShareState(drafts, workspace)[0])).toBe(true) + }) + it('returns the same reference when nothing changes', () => { + const drafts = [item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' })] + expect(mergeShareState(drafts, drafts)).toBe(drafts) + }) + it('leaves drafts without a workspace match untouched', () => { + const drafts = [item({ key: 'app:f/gone', path: 'f/gone', kind: 'app' })] + const merged = mergeShareState(drafts, [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })]) + expect(merged).toBe(drafts) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts new file mode 100644 index 0000000000..669c0a43de --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -0,0 +1,869 @@ +import { describe, it, expect } from 'vitest' +import { + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + buildPathMap, + rewriteContent, + rewriteTriggerConfig, + rewriteFlowValue, + rewriteAppValue, + extractRawAppRefs, + rewriteRawAppContent, + buildProjectBundle, + retargetProjectExport, + collectExportVarPaths, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + type ProjectExport, + type FetchedItem, + type ItemRef +} from './projectBundle' + +describe('classifyPath', () => { + it('internal for paths under the project folder', () => { + expect(classifyPath('f/proj/db', 'proj')).toBe('internal') + expect(classifyPath('f/proj', 'proj')).toBe('internal') + }) + it('hub for hub paths', () => { + expect(classifyPath('hub/16043/discord/send', 'proj')).toBe('hub') + }) + it('external for user and other folders', () => { + expect(classifyPath('u/admin/db', 'proj')).toBe('external') + expect(classifyPath('f/other/db', 'proj')).toBe('external') + }) + it('does not treat a prefix-only match as internal', () => { + expect(classifyPath('f/project2/db', 'proj')).toBe('external') + }) +}) + +describe('extractScriptRefs', () => { + it('finds $res: and res:// resource refs, deduped', () => { + const c = `const a = "$res:u/admin/db"; const b = "res://f/x/api"; const c2 = "$res:u/admin/db"` + expect(extractScriptRefs(c)).toEqual([ + { kind: 'resource', path: 'u/admin/db' }, + { kind: 'resource', path: 'f/x/api' } + ]) + }) + it('returns nothing when no refs', () => { + expect(extractScriptRefs('export async function main() {}')).toEqual([]) + }) +}) + +describe('extractFlowRefs', () => { + it('finds inline-code, static-input, and script-path refs', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { + other: { type: 'static', value: '$res:f/shared/api' }, + expr1: { type: 'javascript', expr: 'flow_input.x' } + } + } + }, + { + id: 'b', + value: { + type: 'branchone', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'd', value: { type: 'script', path: 'hub/123/x/y' } } + ] + } + ], + default: [{ id: 'e', value: { type: 'rawscript', content: 'no refs' } }] + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'f/shared/api' }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/my_script' }) + expect(refs).toContainEqual({ kind: 'script', path: 'hub/123/x/y' }) + // a javascript expr (flow_input) is not a hardcoded ref + expect(refs.filter((r) => r.path === 'flow_input.x')).toEqual([]) + }) + it('finds sub-flow refs from type: flow steps', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }, + { id: 'b', value: { type: 'flow', path: 'hub/9/x/y' } } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sub_flow' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'hub/9/x/y' }) + }) +}) + +describe('buildPathMap', () => { + it('reparents into the project folder keeping the leaf name', () => { + const m = buildPathMap(['u/admin/db', 'f/other/api'], 'proj') + expect(m.get('u/admin/db')).toBe('f/proj/db') + expect(m.get('f/other/api')).toBe('f/proj/api') + }) + it('suffixes collisions deterministically', () => { + const m = buildPathMap(['u/alice/db', 'f/shared/db', 'u/bob/db'], 'proj') + // sorted: f/shared/db, u/alice/db, u/bob/db + expect(m.get('f/shared/db')).toBe('f/proj/db') + expect(m.get('u/alice/db')).toBe('f/proj/db_2') + expect(m.get('u/bob/db')).toBe('f/proj/db_3') + }) + it('maps internal paths to themselves, preserving subfolder depth', () => { + const m = buildPathMap(['f/proj/api', 'f/proj/sub/deep/script'], 'proj') + expect(m.get('f/proj/api')).toBe('f/proj/api') + expect(m.get('f/proj/sub/deep/script')).toBe('f/proj/sub/deep/script') + }) + it('does not flatten two internal items sharing a leaf name', () => { + const m = buildPathMap(['f/proj/a/x', 'f/proj/b/x'], 'proj') + expect(m.get('f/proj/a/x')).toBe('f/proj/a/x') + expect(m.get('f/proj/b/x')).toBe('f/proj/b/x') + }) + it('relocates an external onto a suffix when its leaf collides with an internal path', () => { + const m = buildPathMap(['f/proj/db', 'u/admin/db'], 'proj') + expect(m.get('f/proj/db')).toBe('f/proj/db') + expect(m.get('u/admin/db')).toBe('f/proj/db_2') + }) +}) + +describe('rewriteContent', () => { + it('rewrites mapped refs and leaves unmapped ones', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + expect(rewriteContent('x = "$res:u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "res://u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "$res:hub/1/a/b"', map)).toBe('x = "$res:hub/1/a/b"') + }) + it('does not partial-match a longer path', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + // u/admin/db2 must not be rewritten by the u/admin/db entry + expect(rewriteContent('x = "$res:u/admin/db2"', map)).toBe('x = "$res:u/admin/db2"') + }) +}) + +describe('rewriteTriggerConfig', () => { + const map = new Map([ + ['f/proj/kafka', 'f/target/kafka'], + ['f/proj/script', 'f/target/script'] + ]) + it('remaps plain resource path fields', () => { + expect( + rewriteTriggerConfig({ kafka_resource_path: 'f/proj/kafka', group_id: 'g1' }, map) + ).toEqual({ kafka_resource_path: 'f/target/kafka', group_id: 'g1' }) + }) + it('remaps nested objects, arrays, and $res: tokens', () => { + expect( + rewriteTriggerConfig( + { + nested: { path: 'f/proj/script' }, + list: ['f/proj/kafka', 'unrelated'], + code: 'x = "$res:f/proj/kafka"' + }, + map + ) + ).toEqual({ + nested: { path: 'f/target/script' }, + list: ['f/target/kafka', 'unrelated'], + code: 'x = "$res:f/target/kafka"' + }) + }) + it('leaves non-matching strings and non-string values untouched', () => { + const config = { url: 'wss://example.com', port: 9092, enabled: true, extra: null } + expect(rewriteTriggerConfig(config, map)).toEqual(config) + }) +}) + +describe('rewriteFlowValue', () => { + it('rewrites inline code, static inputs, and script paths; clones input', () => { + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['f/shared/api', 'f/proj/api'], + ['u/admin/my_script', 'f/proj/my_script'] + ]) + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { other: { type: 'static', value: '$res:f/shared/api' } } + } + }, + { id: 'b', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'c', value: { type: 'script', path: 'hub/1/keep/me' } } + ] + } + const out = rewriteFlowValue(value, map) + expect(out.modules[0].value.content).toBe('const db = "$res:f/proj/pg"') + expect(out.modules[0].value.input_transforms.other.value).toBe('$res:f/proj/api') + expect(out.modules[1].value.path).toBe('f/proj/my_script') + expect(out.modules[2].value.path).toBe('hub/1/keep/me') + // original untouched (deep clone) + expect(value.modules[0].value.content).toBe('const db = "$res:u/admin/pg"') + }) +}) + +// A trimmed app value: a runnable-by-path component, a hub runnable, a $res in an +// inline script, and incidental `f/...` text that must NOT be rewritten. +const appValue = () => ({ + grid: [ + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'script', path: 'u/admin/charts' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'flow', path: 'f/shared/sync' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'hubscript', path: 'hub/1/keep' } + } + } + } + ], + hiddenInlineScripts: [ + { name: 'h', inlineScript: { content: 'x = "$res:u/admin/pg"', language: 'deno' } } + ], + someLabel: 'see docs at f/shared/sync for details' +}) + +describe('extractAppRefs', () => { + it('extracts runnable-by-path scripts/flows and $res resources, skips hub', () => { + const refs = extractAppRefs(appValue()) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/charts' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'f/shared/sync' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) +}) + +describe('rewriteAppValue', () => { + it('relocates runnable paths and $res, leaves hub refs and incidental text intact', () => { + const map = new Map([ + ['u/admin/charts', 'f/proj/charts'], + ['f/shared/sync', 'f/proj/sync'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const value = appValue() + const out = rewriteAppValue(value, map) + expect(out.grid[0].data.componentInput.runnable.path).toBe('f/proj/charts') + expect(out.grid[1].data.componentInput.runnable.path).toBe('f/proj/sync') + expect(out.grid[2].data.componentInput.runnable.path).toBe('hub/1/keep') + expect(out.hiddenInlineScripts[0].inlineScript.content).toBe('x = "$res:f/proj/pg"') + // incidental text untouched + expect(out.someLabel).toBe('see docs at f/shared/sync for details') + // original untouched (deep clone) + expect(value.grid[0].data.componentInput.runnable.path).toBe('u/admin/charts') + }) +}) + +describe('raw app (value.raw JSON string)', () => { + const rawContent = () => + JSON.stringify({ + runnables: { + a: { type: 'path', runType: 'flow', path: 'u/admin/sync' }, + b: { type: 'path', runType: 'script', path: 'f/shared/calc' }, + c: { type: 'path', runType: 'hubscript', path: 'hub/1/keep' } + }, + files: { '/bundle.js': 'const conn = "$res:u/admin/pg"' } + }) + + it('extractRawAppRefs sees nested runnables and $res, skips hub', () => { + const refs = extractRawAppRefs(rawContent()) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sync' }) + expect(refs).toContainEqual({ kind: 'script', path: 'f/shared/calc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) + + it('rewriteRawAppContent relocates nested runnable paths and $res', () => { + const map = new Map([ + ['u/admin/sync', 'f/proj/sync'], + ['f/shared/calc', 'f/proj/calc'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const out = JSON.parse(rewriteRawAppContent(rawContent(), map)) + expect(out.runnables.a.path).toBe('f/proj/sync') + expect(out.runnables.b.path).toBe('f/proj/calc') + expect(out.runnables.c.path).toBe('hub/1/keep') + expect(out.files['/bundle.js']).toBe('const conn = "$res:f/proj/pg"') + }) + + it('falls back to $res scan on non-JSON content', () => { + expect(extractRawAppRefs('x = "$res:u/admin/pg"')).toContainEqual({ + kind: 'resource', + path: 'u/admin/pg' + }) + expect( + rewriteRawAppContent('x = "$res:u/admin/pg"', new Map([['u/admin/pg', 'f/proj/pg']])) + ).toBe('x = "$res:f/proj/pg"') + }) +}) + +describe('buildProjectBundle', () => { + // A flow that calls an external script which itself hardcodes a resource. + const flow: FetchedItem = { + kind: 'flow', + path: 'u/admin/my_flow', + summary: 'Flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/helper' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:f/shared/api"', + input_transforms: {} + } + } + ] + } + } + const helper: FetchedItem = { + kind: 'script', + path: 'u/admin/helper', + summary: 'Helper', + language: 'bun', + content: 'const db = "$res:u/admin/pg"; export async function main(){}' + } + + const deps = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/my_flow') return flow + if (ref.path === 'u/admin/helper') return helper + return undefined + }, + resolveResourceType: async (path: string) => { + if (path === 'u/admin/pg') return 'postgresql' + if (path === 'f/shared/api') return 'http_api' + return undefined + } + } + + it('pulls in referenced scripts + resources and rewrites everything under the folder', async () => { + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/my_flow' }], + 'proj', + deps + ) + + // flow + transitively-pulled helper script are both bundled + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + expect(Object.keys(byPath).sort()).toEqual(['u/admin/helper', 'u/admin/my_flow']) + + // items relocated under f/proj/ + expect(byPath['u/admin/my_flow'].newPath).toBe('f/proj/my_flow') + expect(byPath['u/admin/helper'].newPath).toBe('f/proj/helper') + + // flow's script-path ref rewritten to the helper's new path + expect(byPath['u/admin/my_flow'].value.modules[0].value.path).toBe('f/proj/helper') + // flow inline + helper code resource refs rewritten + expect(byPath['u/admin/my_flow'].value.modules[1].value.content).toBe( + 'const x = "$res:f/proj/api"' + ) + expect(byPath['u/admin/helper'].content).toContain('"$res:f/proj/pg"') + + // resource stubs created at new paths with resolved types + const stubs = Object.fromEntries(bundle.resourceStubs.map((s) => [s.originalPath, s])) + expect(stubs['u/admin/pg'].newPath).toBe('f/proj/pg') + expect(stubs['u/admin/pg'].resource_type).toBe('postgresql') + expect(stubs['f/shared/api'].resource_type).toBe('http_api') + + expect(bundle.unresolved).toEqual([]) + }) + + it('pulls in a sub-flow referenced by a type: flow step and rewrites its path', async () => { + const parent: FetchedItem = { + kind: 'flow', + path: 'u/admin/parent_flow', + value: { modules: [{ id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }] } + } + const sub: FetchedItem = { + kind: 'flow', + path: 'u/admin/sub_flow', + value: { + modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/keep/me' } }] + } + } + const d = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/parent_flow') return parent + if (ref.path === 'u/admin/sub_flow') return sub + return undefined + }, + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/parent_flow' }], + 'proj', + d + ) + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + // both flows bundled + expect(Object.keys(byPath).sort()).toEqual(['u/admin/parent_flow', 'u/admin/sub_flow']) + // parent's type: flow ref rewritten to the sub-flow's new path + expect(byPath['u/admin/parent_flow'].value.modules[0].value.path).toBe('f/proj/sub_flow') + expect(byPath['u/admin/sub_flow'].newPath).toBe('f/proj/sub_flow') + // hub ref inside the sub-flow left untouched + expect(byPath['u/admin/sub_flow'].value.modules[0].value.path).toBe('hub/1/keep/me') + expect(bundle.unresolved).toEqual([]) + }) + + it('leaves hub script references untouched and does not fetch them', async () => { + const hubFlow: FetchedItem = { + kind: 'flow', + path: 'u/admin/hub_flow', + value: { modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/x/y' } }] } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/hub_flow' ? hubFlow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/hub_flow' }], 'proj', d) + expect(bundle.items.map((i) => i.path)).toEqual(['u/admin/hub_flow']) + expect(bundle.items[0].value.modules[0].value.path).toBe('hub/1/x/y') + expect(bundle.unresolved).toEqual([]) + }) + + it('reports a missing item and an unresolvable resource as unresolved', async () => { + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/gone' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:u/admin/untyped"', + input_transforms: {} + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved.sort()).toEqual(['u/admin/gone', 'u/admin/untyped']) + }) + + it('relocates $var:/$jsonvar: refs into the slug when it differs from the source folder', async () => { + const flow: FetchedItem = { + kind: 'flow', + path: 'f/source_folder/main', + value: { + flow_env: { CFG: '$jsonvar:f/source_folder/cfg' }, + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + // Whole-value ref is relocated; the inline literal is not. + content: 'return "$var:f/source_folder/key"', + input_transforms: { k: { type: 'static', value: '$var:f/source_folder/key' } } + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'f/source_folder/main' ? flow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'f/source_folder/main' }], + 'kit', + d + ) + const v = bundle.items[0].value + expect(v.modules[0].value.input_transforms.k.value).toBe('$var:f/kit/key') + expect(v.flow_env.CFG).toBe('$jsonvar:f/kit/cfg') + // Inline code literal is untouched. + expect(v.modules[0].value.content).toBe('return "$var:f/source_folder/key"') + }) + + it('dedupes a path missing as both a script and a flow', async () => { + // A missing script + flow sharing a path each push the bare path once; the + // list must stay unique so a keyed UI render of it can't collide. + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/dup' } }, + { id: 'b', value: { type: 'flow', path: 'u/admin/dup' } } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved).toEqual(['u/admin/dup']) + }) +}) + +describe('extractVarRefsFromValue', () => { + it('collects whole-value `$var:`/`$jsonvar:` refs, deduped, walking nested JSON', () => { + const value = { + flow_env: { API: '$var:u/admin/key' }, + modules: [ + { value: { input_transforms: { a: { type: 'static', value: '$var:f/proj/token' } } } }, + { value: { input_transforms: { b: { type: 'static', value: '$jsonvar:u/admin/cfg' } } } }, + { value: { input_transforms: { c: { type: 'static', value: '$var:u/admin/key' } } } } + ] + } + expect(extractVarRefsFromValue(value).sort()).toEqual([ + 'f/proj/token', + 'u/admin/cfg', + 'u/admin/key' + ]) + }) + it('ignores a `$var:` token embedded in inline code (not a whole value)', () => { + // The worker only substitutes a value that *is* the reference, so an inline + // script literal must not be treated as a variable arg. + const value = { + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/example/template"' } }] + } + expect(extractVarRefsFromValue(value)).toEqual([]) + }) +}) + +describe('retargetProjectExport', () => { + const baseExport = (): ProjectExport => ({ + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [ + { + path: 'f/proj/hello', + content: 'const r = "$res:f/proj/db"', + summary: 'hello' + } + ], + flows: [ + { + path: 'f/proj/main_flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'f/proj/hello', input_transforms: {} } } + ] + } + } + ], + apps: [ + { + path: 'f/proj/dashboard', + value: { grid: [{ data: { componentInput: { runnable: {} } } }] } + }, + { + path: 'f/proj/rawapp', + app_type: 'raw', + value: { raw: JSON.stringify({ files: {}, runnables: {} }) } + } + ], + resources: [{ path: 'f/proj/db', resource_type: 'postgresql' }], + triggers: [ + { + path: 'f/proj/every_day', + kind: 'schedule', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { schedule: '0 0 12 * * *' } + }, + { + path: 'f/proj/kafka_in', + kind: 'kafka', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { kafka_resource_path: 'f/proj/db' } + } + ] + }) + + it('returns the bundle unchanged when the folder matches the slug', () => { + const bundle = baseExport() + expect(retargetProjectExport(bundle, 'proj', 'proj')).toBe(bundle) + }) + + it('relocates every item path and internal reference into the target folder', () => { + const out = retargetProjectExport(baseExport(), 'proj', 'dest') + expect(out.scripts[0].path).toBe('f/dest/hello') + expect(out.scripts[0].content).toContain('$res:f/dest/db') + expect(out.flows[0].path).toBe('f/dest/main_flow') + expect(out.flows[0].value.modules[0].value.path).toBe('f/dest/hello') + expect(out.apps.map((a) => a.path)).toEqual(['f/dest/dashboard', 'f/dest/rawapp']) + expect(out.resources[0].path).toBe('f/dest/db') + expect(out.triggers[0].path).toBe('f/dest/every_day') + expect(out.triggers[0].runnable_path).toBe('f/dest/hello') + // Plain-string resource path in a trigger config is remapped too. + expect(out.triggers[1].config.kafka_resource_path).toBe('f/dest/db') + }) + + it('leaves external and hub paths untouched', () => { + const bundle = baseExport() + bundle.scripts[0].content = 'const a = "$res:u/admin/db"; const b = "$res:hub/1/x"' + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.scripts[0].content).toContain('$res:u/admin/db') + expect(out.scripts[0].content).toContain('$res:hub/1/x') + }) + + it('retargets internal $var:/$jsonvar: refs but leaves external ones', () => { + const bundle = baseExport() + bundle.flows[0].value.modules[0].value.input_transforms = { + key: { type: 'static', value: '$var:f/proj/api_key' }, + ext: { type: 'static', value: '$var:u/admin/personal' } + } + bundle.flows[0].value.flow_env = { CFG: '$jsonvar:f/proj/cfg' } + bundle.triggers[1].config.queue_url = '$var:f/proj/sqs' + const out = retargetProjectExport(bundle, 'proj', 'dest') + const it = out.flows[0].value.modules[0].value.input_transforms + expect(it.key.value).toBe('$var:f/dest/api_key') + expect(it.ext.value).toBe('$var:u/admin/personal') + expect(out.flows[0].value.flow_env.CFG).toBe('$jsonvar:f/dest/cfg') + expect(out.triggers[1].config.queue_url).toBe('$var:f/dest/sqs') + }) + + it('leaves an inert $var: literal embedded in inline code unchanged', () => { + const bundle = baseExport() + // Same path as a real runtime ref, but here it is a literal inside code: it + // must not be rewritten even once the path enters the retarget map. + bundle.flows[0].value.modules[0].value = { + type: 'rawscript', + content: 'return "$var:f/proj/api_key"', + input_transforms: { real: { type: 'static', value: '$var:f/proj/api_key' } } + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + const mod = out.flows[0].value.modules[0].value + expect(mod.content).toBe('return "$var:f/proj/api_key"') + expect(mod.input_transforms.real.value).toBe('$var:f/dest/api_key') + }) +}) + +describe('collectExportVarPaths', () => { + it('gathers variable refs from flows, apps, and triggers (deduped)', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [], + flows: [{ path: 'f/proj/f', value: { flow_env: { A: '$var:f/proj/a' }, modules: [] } }], + apps: [ + { + path: 'f/proj/raw', + app_type: 'raw', + value: { raw: JSON.stringify({ runnables: { r: { fields: { x: '$var:u/admin/b' } } } }) } + } + ], + triggers: [{ path: 'f/proj/t', kind: 'sqs', config: { queue_url: '$jsonvar:f/proj/a' } }], + resources: [] + } + expect(collectExportVarPaths(bundle).sort()).toEqual(['f/proj/a', 'u/admin/b']) + }) +}) + +describe('trigger handler relocation', () => { + it('rewriteTriggerConfig remaps script/- and flow/-prefixed handler refs', () => { + const map = new Map([ + ['u/admin/handler', 'f/proj/handler'], + ['u/admin/recovery_flow', 'f/proj/recovery_flow'] + ]) + const out = rewriteTriggerConfig( + { + error_handler_path: 'u/admin/handler', + on_failure: 'script/u/admin/handler', + on_recovery: 'flow/u/admin/recovery_flow', + on_success: 'script/u/admin/unmapped' + }, + map + ) + expect(out.error_handler_path).toBe('f/proj/handler') + expect(out.on_failure).toBe('script/f/proj/handler') + expect(out.on_recovery).toBe('flow/f/proj/recovery_flow') + expect(out.on_success).toBe('script/u/admin/unmapped') + }) + + it('remaps $script:/$flow: only in the url field, never in literal payloads', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + initial_messages: [{ raw_message: '$script:u/admin/builder' }] + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.initial_messages[0].raw_message).toBe('$script:u/admin/builder') + }) + + it('leaves literal handler-shaped strings in args untouched', () => { + const map = new Map([['f/proj/handler', 'f/dest/handler']]) + const out = rewriteTriggerConfig( + { + on_failure: 'script/f/proj/handler', + args: { note: 'script/f/proj/handler' } + }, + map + ) + expect(out.on_failure).toBe('script/f/dest/handler') + expect(out.args.note).toBe('script/f/proj/handler') + }) + + it('leaves nested url keys untouched, rewriting only the top-level websocket url', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + args: { url: '$script:u/admin/builder' } + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.args.url).toBe('$script:u/admin/builder') + }) + + it('extracts and relocates $res refs nested in static input transform JSON', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'script', + path: 'f/proj/step', + input_transforms: { + provider: { type: 'static', value: { resource: '$res:u/admin/openai' } }, + note: { type: 'static', value: 'plain text' } + } + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/openai' }) + const out = rewriteFlowValue(value, new Map([['u/admin/openai', 'f/proj/openai']])) + const it0 = out.modules[0].value.input_transforms + expect(it0.provider.value).toEqual({ resource: '$res:f/proj/openai' }) + expect(typeof it0.note.value).toBe('string') + }) + + it('retargetProjectExport remaps trigger error handlers with the bundle', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [{ path: 'f/proj/handler', content: '' }], + flows: [], + apps: [], + resources: [], + triggers: [ + { + path: 'f/proj/sched', + kind: 'schedule', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { schedule: '0 0 * * * *', on_failure: 'script/f/proj/handler' } + }, + { + path: 'f/proj/mq', + kind: 'mqtt', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { error_handler_path: 'f/proj/handler' } + } + ] + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.triggers[0].config.on_failure).toBe('script/f/dest/handler') + expect(out.triggers[1].config.error_handler_path).toBe('f/dest/handler') + }) +}) + +describe('extractTriggerConfigResourceRefs', () => { + it('collects $res: tokens nested anywhere in a trigger config', () => { + expect( + extractTriggerConfigResourceRefs({ + schedule: '0 0 * * * *', + args: { channel: '$res:u/admin/slack' }, + on_failure_extra_args: { db: 'res://f/other/pg' }, + error_handler_args: { nested: { deep: '$res:u/admin/slack' } } + }) + ).toEqual(['u/admin/slack', 'f/other/pg']) + }) +}) + +describe('flow_env and preprocessor_module', () => { + const flowValue = { + modules: [], + preprocessor_module: { + id: 'pre', + value: { type: 'script', path: 'u/admin/preproc', input_transforms: {} } + }, + flow_env: { SLACK: '$res:u/admin/slack', PLAIN: 'not-a-ref' } + } + + it('walks nested children of the failure module', () => { + const refs = extractFlowRefs({ + modules: [], + failure_module: { + id: 'failure', + value: { + type: 'forloopflow', + modules: [ + { id: 'f-a', value: { type: 'script', path: 'u/admin/cleanup', input_transforms: {} } } + ] + } + } + }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/cleanup' }) + }) + + it('extractFlowRefs sees preprocessor scripts and flow_env resources', () => { + const refs = extractFlowRefs(flowValue) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/preproc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/slack' }) + }) + + it('sees and relocates $res refs nested inside JSON flow_env values', () => { + const value = { + modules: [], + flow_env: { CFG: { db: '$res:u/admin/pg', opts: ['res://u/admin/s3'] } } + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/s3' }) + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['u/admin/s3', 'f/proj/s3'] + ]) + const out = rewriteFlowValue(value, map) + expect(out.flow_env.CFG.db).toBe('$res:f/proj/pg') + expect(out.flow_env.CFG.opts[0]).toBe('$res:f/proj/s3') + }) + + it('rewriteFlowValue relocates both', () => { + const map = new Map([ + ['u/admin/preproc', 'f/proj/preproc'], + ['u/admin/slack', 'f/proj/slack'] + ]) + const out = rewriteFlowValue(flowValue, map) + expect(out.preprocessor_module.value.path).toBe('f/proj/preproc') + expect(out.flow_env.SLACK).toBe('$res:f/proj/slack') + expect(out.flow_env.PLAIN).toBe('not-a-ref') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts new file mode 100644 index 0000000000..1fd62498d5 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -0,0 +1,652 @@ +// Pure logic for the "project = folder" Hub bundle. A project is one folder +// `f//...`. Bundling: collect the transitive closure, relocate external +// refs (`u//`, `f//` -> `f//`, `_2`/`_3`… +// on collision) and rewrite them. Hub refs stay external; runtime string-concat +// paths are out of scope. No API/Svelte deps so it's unit-testable. + +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { isRunnableByPath } from '$lib/components/apps/inputType' + +export type RefKind = 'resource' | 'script' | 'flow' + +export interface Ref { + kind: RefKind + /** Bare path, without the `$res:` / `res://` prefix for resources. */ + path: string +} + +export type PathClass = 'internal' | 'hub' | 'external' + +/** A single `$res:PATH` / `res://PATH` token (path captured in group 1). */ +const RES_TOKEN_RE = /(?:\$res:|res:\/\/)([\w\-./]+)/g + +// A whole-string `$var:PATH` / `$jsonvar:PATH` value. The worker substitutes these +// only when an argument value *is* the reference (walking nested JSON), never a +// token embedded in inline code, so the whole value must match. `_KIND` captures +// the prefix (group 1) and path (group 2) so a rewrite can preserve `var`/`jsonvar`. +const VAR_VALUE_RE = /^\$(?:json)?var:([\w\-./]+)$/ +const VAR_VALUE_RE_KIND = /^\$(var|jsonvar):([\w\-./]+)$/ + +// Variable paths a value will resolve at runtime (flow static inputs, flow_env, +// app runnable inputs, trigger config fields). Walk the parsed structure and match +// whole string values so inline code carrying a literal `$var:` string is ignored. +export function extractVarRefsFromValue(value: any): string[] { + const out = new Set() + const walk = (v: any) => { + if (typeof v === 'string') { + const m = VAR_VALUE_RE.exec(v) + if (m) out.add(m[1]) + } else if (Array.isArray(v)) { + for (const x of v) walk(x) + } else if (v && typeof v === 'object') { + for (const k of Object.keys(v)) walk(v[k]) + } + } + walk(value) + return [...out] +} + +export function classifyPath(path: string, slug: string): PathClass { + if (path.startsWith(`f/${slug}/`) || path === `f/${slug}`) return 'internal' + if (path.startsWith('hub/')) return 'hub' + return 'external' +} + +export function extractScriptRefs(content: string): Ref[] { + const out: Ref[] = [] + const seen = new Set() + let m: RegExpExecArray | null + RES_TOKEN_RE.lastIndex = 0 + while ((m = RES_TOKEN_RE.exec(content)) !== null) { + if (!seen.has(m[1])) { + seen.add(m[1]) + out.push({ kind: 'resource', path: m[1] }) + } + } + return out +} + +/** + * References inside a flow value: + * - inline rawscript code with `$res:` (resource) + * - static step inputs whose value is a `$res:` literal (resource) + * - `type: script` steps that reference a script by path (script) + * - `type: flow` steps that reference a sub-flow by path (flow) + */ +export function extractFlowRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + // getAllModules flattens the whole tree (loops, branches, aiagent tools, + // failure module) so each module only needs local inspection; the + // preprocessor module sits outside `modules` and is walked the same way. + for (const mod of allFlowModules(value)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if (v.type === 'script' && typeof v.path === 'string') add('script', v.path) + if (v.type === 'flow' && typeof v.path === 'string') add('flow', v.path) + if (typeof v.content === 'string') { + for (const r of extractScriptRefs(v.content)) add('resource', r.path) + } + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Static values can be a bare `$res:` string or arbitrary JSON with + // refs nested anywhere — the worker resolves both, so scan the full + // serialization. + if (t?.type === 'static' && t.value !== undefined) { + const text = typeof t.value === 'string' ? t.value : JSON.stringify(t.value) + for (const r of extractScriptRefs(text)) add('resource', r.path) + } + } + } + } + // flow_env values support `$res:path` references — as whole string values or + // nested inside JSON values (the worker resolves both), so scan the full + // serialization. + if (value?.flow_env && typeof value.flow_env === 'object') { + for (const r of extractScriptRefs(JSON.stringify(value.flow_env))) add('resource', r.path) + } + return out +} + +// Every module of a flow value: the tree under `modules`, the failure module, +// and the preprocessor module (which lives outside `modules`). Any walk over a +// flow's modules must go through this — a walk that misses a module class +// silently drops its dependencies from bundles or migrations. All three go in +// the root list (not getAllModules' failure_module parameter, which appends +// the module without expanding its descendants) so nested children of a +// failure or preprocessor module are walked too. +export function allFlowModules(value: any) { + return getAllModules([ + ...(value?.modules ?? []), + ...(value?.preprocessor_module ? [value.preprocessor_module] : []), + ...(value?.failure_module ? [value.failure_module] : []) + ]) +} + +// Visit every object node in an app value tree (JSON-safe, no cycles). +function walkAppNodes(value: any, visit: (node: Record) => void): void { + if (value == null || typeof value !== 'object') return + if (Array.isArray(value)) { + for (const v of value) walkAppNodes(v, visit) + return + } + visit(value) + for (const k of Object.keys(value)) walkAppNodes(value[k], visit) +} + +// `runnableByPath`/`path` nodes reference a workspace runnable by path. +function runnableRef(node: Record): Ref | undefined { + if (!isRunnableByPath(node as any) || typeof node.path !== 'string') return undefined + if (node.runType === 'flow') return { kind: 'flow', path: node.path } + if (node.runType === 'script') return { kind: 'script', path: node.path } + return undefined // hubscript -> external hub, ignored +} + +// App refs: `$res:` resources anywhere in the value, plus script/flow runnables +// referenced by path in components. +export function extractAppRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + walkAppNodes(value, (node) => { + const r = runnableRef(node) + if (r) add(r.kind, r.path) + }) + for (const r of extractScriptRefs(JSON.stringify(value ?? {}))) add('resource', r.path) + return out +} + +/** + * Build the relocation map. Internal paths (`f//...`) map to themselves + * and are reserved first; external paths relocate to `f//` (`_2`/`_3`… + * on collision). Input is sorted so suffix assignment is deterministic. + */ +export function buildPathMap(paths: Iterable, slug: string): Map { + const map = new Map() + const used = new Set() + const sorted = [...new Set(paths)].sort() + for (const p of sorted) { + if (classifyPath(p, slug) === 'internal') { + map.set(p, p) + used.add(p) + } + } + for (const old of sorted) { + if (map.has(old)) continue + const name = old.split('/').filter(Boolean).pop() ?? old + let candidate = `f/${slug}/${name}` + let n = 2 + while (used.has(candidate)) candidate = `f/${slug}/${name}_${n++}` + used.add(candidate) + map.set(old, candidate) + } + return map +} + +// Both ref forms normalize to `$res:` on rewrite. +export function rewriteContent(content: string, map: Map): string { + return content.replace(RES_TOKEN_RE, (whole, path) => { + const next = map.get(path) + return next ? `$res:${next}` : whole + }) +} + +// Structurally relocate whole-string `$var:`/`$jsonvar:` values — the only form the +// worker resolves. Walks the parsed value so an inert token embedded in inline code +// or arbitrary text is left untouched, unlike token replacement over serialized +// strings. Only paths present in the map move (the retarget map carries variables). +export function rewriteVarRefsInValue(value: any, map: Map): any { + if (typeof value === 'string') { + const m = VAR_VALUE_RE_KIND.exec(value) + if (m) { + const next = map.get(m[2]) + if (next) return `$${m[1]}:${next}` + } + return value + } + if (Array.isArray(value)) return value.map((v) => rewriteVarRefsInValue(v, map)) + if (value && typeof value === 'object') { + const out: Record = {} + for (const k of Object.keys(value)) out[k] = rewriteVarRefsInValue(value[k], map) + return out + } + return value +} + +/** + * `$res:`/`res://` tokens anywhere in a trigger config — schedule args, + * on_*_extra_args, error_handler_args, … (e.g. the built-in Slack handler + * stores its channel resource this way). These must enter the bundle path map + * so `rewriteTriggerConfig` relocates them and a stub is exported. + */ +export function extractTriggerConfigResourceRefs(config: any): string[] { + return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) +} + +/** + * Trigger configs reference resources as plain path strings (e.g. + * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting + * misses them. Deep-walk the config and remap any string that exact-matches a + * map key (map keys are full bundle paths, so an exact match is a reference), + * or a `script/`/`flow/` handler reference (schedules' on_failure + * et al.), falling back to `$res:` token rewriting for embedded refs. + */ +// Top-level config fields whose string values are prefixed runnable refs. +// Prefixed forms are remapped ONLY in these known positions: deciding meaning +// from string shape alone rewrote literal payloads that merely looked like +// refs. Bare-path exact matches and $res: tokens stay position-independent. +const HANDLER_REF_FIELDS = new Set(['on_failure', 'on_recovery', 'on_success']) + +export function rewriteTriggerConfig(config: any, map: Map, depth = 0): any { + if (typeof config === 'string') { + const direct = map.get(config) + if (direct) return direct + return rewriteContent(config, map) + } + if (Array.isArray(config)) return config.map((v) => rewriteTriggerConfig(v, map, depth + 1)) + if (config && typeof config === 'object') { + return Object.fromEntries( + Object.entries(config).map(([k, v]) => { + if (depth === 0 && typeof v === 'string') { + // Websocket url: $script: / $flow:. + if (k === 'url') { + const m = /^\$(script|flow):(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `$${m[1]}:${map.get(m[2])}`] + } + // Schedule handlers: script/ / flow/. + if (HANDLER_REF_FIELDS.has(k)) { + const m = /^(script|flow)\/(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `${m[1]}/${map.get(m[2])}`] + } + } + return [k, rewriteTriggerConfig(v, map, depth + 1)] + }) + ) + } + return config +} + +export function rewriteFlowValue(value: any, map: Map): any { + const cloned = JSON.parse(JSON.stringify(value ?? {})) + for (const mod of allFlowModules(cloned)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if ( + (v.type === 'script' || v.type === 'flow') && + typeof v.path === 'string' && + map.has(v.path) + ) { + v.path = map.get(v.path) + } + if (typeof v.content === 'string') v.content = rewriteContent(v.content, map) + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Mirror extraction: rewrite refs wherever they sit, preserving the + // value's type (a string stays a string, JSON round-trips). + if (t?.type === 'static' && t.value !== undefined) { + if (typeof t.value === 'string') { + t.value = rewriteContent(t.value, map) + } else { + t.value = JSON.parse(rewriteContent(JSON.stringify(t.value), map)) + } + } + } + } + } + if (cloned?.flow_env && typeof cloned.flow_env === 'object') { + // Tokens can sit inside nested JSON values, not just string values; the + // serialize→rewrite→parse round-trip reaches all of them (paths contain + // no characters that would break JSON string literals). + cloned.flow_env = JSON.parse(rewriteContent(JSON.stringify(cloned.flow_env), map)) + } + return cloned +} + +// Relocate `$res:` tokens (one round-trip, also produces a fresh clone) then +// runnable-by-path refs structurally. Incidental `f//` strings stay intact. +export function rewriteAppValue(value: any, map: Map): any { + if (value == null) return value + const cloned = JSON.parse(rewriteContent(JSON.stringify(value), map)) + walkAppNodes(cloned, (node) => { + if (runnableRef(node) && map.has(node.path)) node.path = map.get(node.path) + }) + return cloned +} + +// Raw/compiled apps store their structure as a JSON string (`{ runnables, files }`). +// Parse it so runnable-by-path refs in the runnables map are seen, reusing the +// same walk; fall back to plain `$res:` scanning if it isn't valid JSON. +export function extractRawAppRefs(content: string): Ref[] { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return extractScriptRefs(content) + } + return extractAppRefs(parsed) +} + +export function rewriteRawAppContent(content: string, map: Map): string { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return rewriteContent(content, map) + } + return JSON.stringify(rewriteAppValue(parsed, map)) +} + +// --------------------------------------------------------------------------- +// Hub project export format (what /projects/{slug}/export returns) and its +// retargeting into a destination folder. Kept here, next to the rewriters, +// so the bundle format is defined in one module for both publish and install. +// --------------------------------------------------------------------------- + +export type ExportItem = Record +export interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean +} +export interface ProjectExport { + project: { slug: string; name: string; summary: string; readme: string | null } + scripts: ExportItem[] + flows: ExportItem[] + apps: ExportItem[] + resources: ExportItem[] + triggers: ExportItem[] + migrations?: ProjectMigration[] +} + +// Map bundled paths `f//...` -> `f//...`. Only enumerated +// paths go in, so rewriters touch real refs, never incidental text. +export function buildRetargetMap( + bundle: ProjectExport, + fromSlug: string, + folder: string +): Map { + const map = new Map() + const prefix = `f/${fromSlug}/` + const add = (p: unknown) => { + if (typeof p === 'string' && p.startsWith(prefix)) { + map.set(p, `f/${folder}/${p.slice(prefix.length)}`) + } + } + for (const s of bundle.scripts) add(s.path) + for (const f of bundle.flows) add(f.path) + for (const a of bundle.apps) add(a.path) + for (const r of bundle.resources) add(r.path) + for (const t of bundle.triggers) { + add(t.path) + add(t.runnable_path) + } + // Variables aren't enumerated in the export; their `$var:`/`$jsonvar:` refs live + // inside item values. Relocate the internal ones so a renamed-folder import + // rewrites them into the target folder instead of retaining the old prefix. + for (const p of collectExportVarPaths(bundle)) add(p) + return map +} + +// Internal-or-external variable paths referenced by the export's flows, apps and +// triggers. Scripts carry no variable args. Raw apps hold their structure in the +// `value.raw` JSON string. +export function collectExportVarPaths(bundle: ProjectExport): string[] { + const out = new Set() + const collect = (value: any) => { + for (const p of extractVarRefsFromValue(value)) out.add(p) + } + for (const f of bundle.flows) collect(f.value) + for (const a of bundle.apps) collect(a.app_type === 'raw' ? safeParseRaw(a.value?.raw) : a.value) + for (const t of bundle.triggers) collect(t.config) + return [...out] +} + +function safeParseRaw(raw: unknown): any { + if (typeof raw !== 'string') return undefined + try { + return JSON.parse(raw) + } catch { + return undefined + } +} + +// Structural retarget: rewrite each item's path and its internal refs, +// leaving Hub refs and arbitrary content untouched. +export function retargetProjectExport( + bundle: ProjectExport, + fromSlug: string, + folder: string +): ProjectExport { + if (folder === fromSlug) return bundle + const map = buildRetargetMap(bundle, fromSlug, folder) + const remap = (p: unknown) => (typeof p === 'string' ? (map.get(p) ?? p) : p) + return { + ...bundle, + scripts: bundle.scripts.map((s) => ({ + ...s, + path: remap(s.path), + content: rewriteContent(s.content ?? '', map) + })), + flows: bundle.flows.map((f) => ({ + ...f, + path: remap(f.path), + value: rewriteVarRefsInValue(rewriteFlowValue(f.value, map), map) + })), + apps: bundle.apps.map((a) => ({ + ...a, + path: remap(a.path), + // Raw apps keep their structure in the `value.raw` JSON string. + value: + a.app_type === 'raw' + ? { + ...a.value, + raw: rewriteRawVarRefs(rewriteRawAppContent(a.value?.raw ?? '', map), map) + } + : rewriteVarRefsInValue(rewriteAppValue(a.value, map), map) + })), + resources: bundle.resources.map((r) => ({ ...r, path: remap(r.path) })), + triggers: bundle.triggers.map((t) => ({ + ...t, + path: remap(t.path), + runnable_path: remap(t.runnable_path), + // Configs hold `$res:` tokens, plain resource paths (kafka_resource_path + // etc.) and whole-string `$var:` values — rewrite all three. + config: t.config ? rewriteVarRefsInValue(rewriteTriggerConfig(t.config, map), map) : t.config + })) + } +} + +// Var relocation for a raw app's `value.raw` JSON string: parse, structurally +// rewrite whole-string var values, re-serialize; leave invalid JSON untouched. +function rewriteRawVarRefs(raw: string, map: Map): string { + const parsed = safeParseRaw(raw) + if (parsed === undefined) return raw + return JSON.stringify(rewriteVarRefsInValue(parsed, map)) +} + +export type ItemKind = 'script' | 'flow' | 'app' | 'raw_app' + +export interface ItemRef { + kind: ItemKind + path: string +} + +export interface FetchedItem { + kind: ItemKind + path: string + summary?: string + description?: string + /** scripts + raw_apps */ + content?: string + /** flows + apps */ + value?: any + /** scripts */ + language?: string + schema?: any + lock?: string + scriptKind?: string +} + +export interface BundleDeps { + /** Fetch a workspace item by ref, or undefined if it doesn't exist. */ + fetchItem: (ref: ItemRef) => Promise + /** Resolve a resource path to its type, or undefined if missing. */ + resolveResourceType: (path: string) => Promise +} + +export interface BundledItem extends FetchedItem { + /** Path the item takes inside the project folder. */ + newPath: string +} + +export interface ResourceStub { + originalPath: string + newPath: string + resource_type: string +} + +export interface ProjectBundle { + items: BundledItem[] + resourceStubs: ResourceStub[] + /** Original -> relocated path for every item and resource (incl. unresolved). */ + pathMap: Map + /** External paths we couldn't fetch/resolve (missing items or untyped resources). */ + unresolved: string[] +} + +function refsForFetched(item: FetchedItem): Ref[] { + if (item.kind === 'script') return extractScriptRefs(item.content ?? '') + if (item.kind === 'flow') return extractFlowRefs(item.value) + if (item.kind === 'app') return extractAppRefs(item.value) + if (item.kind === 'raw_app') return extractRawAppRefs(item.content ?? '') + return [] +} + +// Whole-string `$var:`/`$jsonvar:` paths an item resolves at runtime. Scripts carry +// no variable args; raw apps hold their structure in the `content` JSON string. +function varRefsForFetched(item: FetchedItem): string[] { + if (item.kind === 'flow' || item.kind === 'app') return extractVarRefsFromValue(item.value) + if (item.kind === 'raw_app') return extractVarRefsFromValue(safeParseRaw(item.content)) + return [] +} + +// Walks the transitive closure: scripts referenced by path are pulled in +// recursively, resources become empty stubs, hub refs stay external. +export async function buildProjectBundle( + seed: ItemRef[], + slug: string, + deps: BundleDeps, + extraResourcePaths: string[] = [], + extraVarPaths: string[] = [] +): Promise { + const fetched = new Map() + const queued = new Set() + const resourcePaths = new Set() + const varPaths = new Set() + const unresolved: string[] = [] + + // Resources and variables referenced by triggers (by config value, not `$res:` + // in code) — relocated through the same map so the export stays slug-relative. + for (const p of extraResourcePaths) { + if (classifyPath(p, slug) !== 'hub') resourcePaths.add(p) + } + for (const p of extraVarPaths) varPaths.add(p) + + // Key by `${kind}:${path}`, not bare path: a script and flow can share a path, + // and keying by path alone would silently drop one. + const refKey = (kind: string, path: string) => `${kind}:${path}` + + // Refs at the same BFS depth are independent: fetch each level concurrently. + let level: ItemRef[] = [] + for (const s of seed) { + const key = refKey(s.kind, s.path) + if (!queued.has(key)) { + queued.add(key) + level.push(s) + } + } + while (level.length > 0) { + const results = await Promise.all( + level.map(async (ref) => ({ ref, item: await deps.fetchItem(ref) })) + ) + const next: ItemRef[] = [] + for (const { ref, item } of results) { + if (!item) { + unresolved.push(ref.path) + continue + } + fetched.set(refKey(ref.kind, ref.path), item) + for (const r of refsForFetched(item)) { + if (classifyPath(r.path, slug) === 'hub') continue + if (r.kind === 'resource') { + resourcePaths.add(r.path) + } else if (r.kind === 'script' || r.kind === 'flow') { + const key = refKey(r.kind, r.path) + if (!queued.has(key)) { + queued.add(key) + next.push({ kind: r.kind, path: r.path }) + } + } + } + // Relocate the item's runtime variable refs into the project folder too, so + // the export is slug-relative regardless of the source folder (import then + // materializes them as placeholders). Variables are never hub-hosted. + for (const p of varRefsForFetched(item)) varPaths.add(p) + } + level = next + } + + const fetchedItems = [...fetched.values()] + const itemPaths = fetchedItems.map((it) => it.path) + const map = buildPathMap([...itemPaths, ...resourcePaths, ...varPaths], slug) + + const items: BundledItem[] = fetchedItems.map((it) => { + const rewritten: BundledItem = { ...it, newPath: map.get(it.path) ?? it.path } + if (it.kind === 'script') { + rewritten.content = rewriteContent(it.content ?? '', map) + } else if (it.kind === 'raw_app') { + rewritten.content = rewriteRawVarRefs(rewriteRawAppContent(it.content ?? '', map), map) + } else if (it.kind === 'flow') { + rewritten.value = rewriteVarRefsInValue(rewriteFlowValue(it.value, map), map) + } else if (it.kind === 'app') { + rewritten.value = rewriteVarRefsInValue(rewriteAppValue(it.value, map), map) + } + return rewritten + }) + + const resourceStubs: ResourceStub[] = [] + const resolved = await Promise.all( + [...resourcePaths].map(async (path) => ({ path, type: await deps.resolveResourceType(path) })) + ) + for (const { path, type } of resolved) { + if (!type) { + unresolved.push(path) + continue + } + resourceStubs.push({ originalPath: path, newPath: map.get(path) ?? path, resource_type: type }) + } + + // `unresolved` keys missing items by kind:path but stores the bare path, so a + // missing script and flow (or a runnable and resource) sharing a path can push + // the same string twice. Dedupe: callers use it as a display/blocker list where + // duplicate keys would break keyed rendering. + return { items, resourceStubs, pathMap: map, unresolved: [...new Set(unresolved)] } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts new file mode 100644 index 0000000000..a02d4e264d --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest' +import { refContainmentViolation, varContainmentViolation } from './projectInstall' +import type { Ref } from './projectBundle' + +describe('refContainmentViolation', () => { + const folder = 'proj' + const violation = (r: Ref) => refContainmentViolation([r], folder) + + it('allows references relocated into the target folder', () => { + expect(violation({ kind: 'resource', path: 'f/proj/db' })).toBeUndefined() + expect(violation({ kind: 'script', path: 'f/proj/helper' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'f/proj/sub' })).toBeUndefined() + }) + + it('allows hub script/flow references but never hub resources', () => { + expect(violation({ kind: 'script', path: 'hub/1/x/y' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'hub/1/a/b' })).toBeUndefined() + // Resources are not hub-hosted, so a hub/ resource path is still an escape. + expect(violation({ kind: 'resource', path: 'hub/1/x/y' })).toBeDefined() + }) + + it('rejects references bound to another namespace', () => { + // The crux: an in-folder runnable pointing its resource at an existing asset. + expect(violation({ kind: 'resource', path: 'u/admin/db' })).toContain('escapes') + expect(violation({ kind: 'script', path: 'f/other/helper' })).toContain('escapes') + expect(violation({ kind: 'flow', path: 'u/admin/sub' })).toContain('escapes') + }) + + it('does not treat a prefix-only folder match as internal', () => { + expect(violation({ kind: 'script', path: 'f/proj2/helper' })).toContain('escapes') + }) + + it('reports the first offending reference and passes a fully-contained set', () => { + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'script', path: 'hub/1/x/y' } + ], + folder + ) + ).toBeUndefined() + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'resource', path: 'u/admin/secret' } + ], + folder + ) + ).toContain('u/admin/secret') + }) +}) + +describe('varContainmentViolation', () => { + const folder = 'proj' + + it('allows in-folder variable references', () => { + expect(varContainmentViolation({ token: '$var:f/proj/token' }, folder)).toBeUndefined() + expect(varContainmentViolation({ x: 'no refs here' }, folder)).toBeUndefined() + }) + + it('rejects a `$var:` or `$jsonvar:` bound to another namespace', () => { + // The crux: a variable arg the ref extractors miss, resolved under the perms. + expect(varContainmentViolation({ queue_url: '$var:u/admin/token' }, folder)).toContain( + 'u/admin/token' + ) + expect(varContainmentViolation({ cfg: '$jsonvar:f/other/secret' }, folder)).toContain('escapes') + }) + + it('ignores a `$var:` literal embedded in inline code', () => { + const flowValue = { + flow_env: { API: '$var:f/proj/api_key' }, + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/admin/should_not_flag"' } }] + } + expect(varContainmentViolation(flowValue, folder)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts new file mode 100644 index 0000000000..1827e00283 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -0,0 +1,405 @@ +// Imports a Hub project export into a workspace: one importer per item kind, +// each item reported individually so one bad item never aborts the rest. +// UI-free — the install page owns folder choice and migration review. + +import { + AppService, + FlowService, + FolderService, + ResourceService, + ScriptService, + VariableService, + WorkspaceService +} from '$lib/gen' +import { + TRIGGER_KINDS, + createWorkspaceTriggerDisabled, + triggerHandlerRefs, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { + classifyPath, + collectExportVarPaths, + extractAppRefs, + extractFlowRefs, + extractRawAppRefs, + extractScriptRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + retargetProjectExport, + type ExportItem, + type ProjectExport, + type ProjectMigration, + type Ref +} from './projectBundle' + +export interface InstallResult { + path: string + ok: boolean + error?: string +} + +// Guarding an item's own path is not enough: the `$res:`/script/flow refs baked +// into its content are live bindings the backend acts on. A well-formed export +// relocates them all into f// (hub/ script refs stay external); anything +// else points a runnable at an existing asset in another namespace, so refuse the +// item rather than bind it there. Resources are never hub-hosted, so a hub/ path +// there is not a valid escape hatch. Mirrors the trigger-config containment. +export function refContainmentViolation(refs: Ref[], folder: string): string | undefined { + for (const r of refs) { + const cls = classifyPath(r.path, folder) + if (cls === 'internal') continue + if (cls === 'hub' && r.kind !== 'resource') continue + return `reference '${r.path}' escapes the target folder f/${folder}/ — skipped` + } + return undefined +} + +// `$var:`/`$jsonvar:` references (in flow static inputs, flow_env, app runnable +// inputs, trigger config) are resolved at runtime under the imported runnable's +// permissions and are never hub-hosted. Retargeting relocates a project's own refs +// into the target folder; anything still outside it points at another namespace, so +// reject those. Takes the parsed value so inline code carrying a literal is ignored. +export function varContainmentViolation(value: any, folder: string): string | undefined { + for (const p of extractVarRefsFromValue(value)) { + if (classifyPath(p, folder) !== 'internal') { + return `variable '${p}' escapes the target folder f/${folder}/ — skipped` + } + } + return undefined +} + +// Surface the backend's explanation: API errors carry the real message in +// `.body` (plain text for Windmill 4xx), while `.message` is the generic +// status text ("Bad Request"). Prefer the body so e.g. a path/route_path +// collision reads as the actual reason, not just "Bad Request". +function errorMessage(e: any): string { + const body = e?.body + if (typeof body === 'string' && body.trim() !== '') return body + if (body && typeof body === 'object') + return body.error?.message ?? body.message ?? JSON.stringify(body) + return e?.message ?? String(e) +} + +// Recompute an app's execution policy from its (retargeted) value, mirroring +// what the editor does on deploy. `triggerables_v2` is keyed by +// `:rawscript/`; retargeting rewrites that +// content, so a copied or empty policy would leave every inline runnable +// "forbidden by policy" at runtime. Default to publisher (auth required). +async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} +async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} + +function importScript(workspace: string, s: ExportItem): Promise { + return ScriptService.createScript({ + workspace, + requestBody: { + path: s.path, + summary: s.summary ?? '', + description: s.description ?? '', + content: s.content ?? '', + language: s.language, + schema: s.schema ?? undefined, + kind: s.kind ?? 'script', + lock: s.lockfile ?? undefined + } + }) +} + +function importFlow(workspace: string, f: ExportItem): Promise { + return FlowService.createFlow({ + workspace, + requestBody: { + path: f.path, + summary: f.summary ?? '', + description: f.description ?? '', + value: f.value, + schema: f.schema ?? undefined + } + }) +} + +// Stubs only: never overwrite an existing resource's value (updateIfExists +// stays false so a path collision is reported as a failed item instead). +function importResourceStub(workspace: string, r: ExportItem): Promise { + return ResourceService.createResource({ + workspace, + updateIfExists: false, + requestBody: { + path: r.path, + resource_type: r.resource_type, + value: {}, + description: 'Imported stub — fill in the value.' + } + }) +} + +// Variables hold secrets/config, so their values are never shipped. Create an empty +// secret placeholder for a project variable the importer must fill, mirroring the +// resource stubs. Conflict-safe: an already-present variable (the importer filled it, +// or a re-import) is left untouched rather than clobbered. +async function importVariablePlaceholder(workspace: string, path: string): Promise { + if (await VariableService.existsVariable({ workspace, path })) return + await VariableService.createVariable({ + workspace, + requestBody: { + path, + value: '', + is_secret: true, + description: 'Imported placeholder — fill in the value.' + } + }) +} + +async function importApp(workspace: string, a: ExportItem): Promise { + if (a.app_type === 'raw') { + let parsed: any + try { + parsed = JSON.parse(a.value?.raw ?? '{}') + } catch (e: any) { + throw new Error(`invalid raw app bundle: ${e?.message ?? String(e)}`) + } + const files = { ...(parsed.files ?? {}) } + const js = files['/bundle.js'] ?? '' + const css = files['/bundle.css'] ?? '' + delete files['/bundle.js'] + delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} + return AppService.createAppRaw({ + workspace, + formData: { + app: { + path: a.path, + summary: a.summary ?? '', + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) + }, + js, + css + } + }) + } + return AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }) +} + +// Apply one migration to the target data table. If the data table opted into +// migrations, record it (datatable_migrations + _wm_migrations, run only this +// version); otherwise run the SQL once as a preview job (unrecorded). +async function applyOneMigration( + workspace: string, + projectSlug: string, + m: ProjectMigration +): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${projectSlug}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } +} + +/** + * Install a project export into `workspace` under `f//`: create the + * folder, retarget every item, import kind by kind, then apply the (already + * reviewed) migrations. Each item's outcome is reported through `onResult`; + * failures never abort the remaining items. + */ +export async function installProject(args: { + workspace: string + exportData: ProjectExport + folder: string + migrations: ProjectMigration[] + hasEeLicense: boolean + onResult: (r: InstallResult) => void +}): Promise { + const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + + const record = (path: string, p: Promise): Promise => + p.then( + () => onResult({ path, ok: true }), + (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) + ) + + try { + await FolderService.createFolder({ workspace, requestBody: { name: folder } }) + } catch {} + + const proj = retargetProjectExport(exportData, exportData.project.slug, folder) + + // The export is remote input: every path it wants to write must stay inside + // the folder the user chose. Anything else (crafted export, or an export + // whose items weren't relocated into f// at publish) is refused + // per-item instead of being created in another namespace. + const prefix = `f/${folder}/` + const guard = (path: unknown, ...also: unknown[]): string | undefined => { + for (const p of [path, ...also]) { + if (typeof p !== 'string' || !p.startsWith(prefix)) { + return `path '${String(p)}' escapes the target folder ${prefix} — skipped` + } + } + return undefined + } + const checked = (path: unknown, run: () => Promise, ...also: unknown[]) => { + const violation = guard(path, ...also) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + // `refs` catches structured runnable/`$res:` refs; `varValue` is the parsed item + // walked for `$var:`/`$jsonvar:` argument refs (which the ref extractors miss). + const checkedItem = (path: unknown, refs: Ref[], varValue: any, run: () => Promise) => { + const violation = + guard(path) ?? + refContainmentViolation(refs, folder) ?? + varContainmentViolation(varValue, folder) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + for (const s of proj.scripts) { + // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), + // not in script source, so there is no variable arg to contain here. + await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => + importScript(workspace, s) + ) + } + for (const f of proj.flows) { + await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) + } + for (const r of proj.resources) { + await checked(r.path, () => importResourceStub(workspace, r)) + } + // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted + // into this folder). External refs are rejected per-item, so only stub in-folder + // ones; guard again in case an out-of-folder ref slipped through retargeting. + for (const p of collectExportVarPaths(proj)) { + if (!p.startsWith(prefix)) continue + await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) + } + for (const a of proj.apps) { + const isRaw = a.app_type === 'raw' + const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) + // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the + // walk sees the same structure the backend resolves. Malformed raw fails at import. + let varValue: any = a.value + if (isRaw) { + try { + varValue = JSON.parse(a.value?.raw ?? '{}') + } catch { + varValue = undefined + } + } + await checkedItem(a.path, refs, varValue, () => importApp(workspace, a)) + } + // A trigger's config is a live binding, not inert content: resource fields, + // handler runnables and $res: refs it names are acted on by the backend, so + // every one must stay inside the chosen folder (handlers may also point at + // hub/ scripts). Otherwise a crafted export could bind the trigger to + // existing assets in another namespace. + const triggerConfigViolation = (t: ExportItem): string | undefined => { + const cfg = (t.config ?? {}) as Record + for (const r of triggerHandlerRefs({ kind: t.kind, config: cfg } as WorkspaceTrigger)) { + if (!r.path.startsWith(prefix) && !r.path.startsWith('hub/')) { + return `handler '${r.path}' escapes the target folder ${prefix} — skipped` + } + } + const resourceRefs = new Set(extractTriggerConfigResourceRefs(cfg)) + const field = TRIGGER_KINDS[t.kind as WorkspaceTriggerKind]?.resourceField + const fieldValue = field ? cfg[field] : undefined + if (typeof fieldValue === 'string' && fieldValue !== '') resourceRefs.add(fieldValue) + for (const p of resourceRefs) { + if (!p.startsWith(prefix)) { + return `resource '${p}' escapes the target folder ${prefix} — skipped` + } + } + // Config fields (e.g. SQS queue_url) can carry `$var:`/`$jsonvar:` refs too. + return varContainmentViolation(cfg, folder) + } + for (const t of proj.triggers) { + const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) + await record( + String(t.path), + violation + ? Promise.reject(new Error(violation)) + : createWorkspaceTriggerDisabled( + workspace, + { + kind: t.kind, + path: t.path, + script_path: t.runnable_path, + is_flow: t.runnable_kind === 'flow', + summary: t.summary ?? null, + config: t.config ?? null + }, + { hasEeLicense } + ) + ) + } + + // Apply the reviewed data table migrations after items exist. + for (const m of migrations) { + await record( + `data table: ${m.datatable_name}`, + applyOneMigration(workspace, exportData.project.slug, m) + ) + } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..7d35c51650 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('collects datatable refs from the preprocessor module', async () => { + inferAssetsMock.mockResolvedValue({ status: 'ok', assets: [] }) + const items: FetchedItem[] = [ + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [], + preprocessor_module: { + id: 'pre', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/inbox' }] + } + } + } + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])]).toEqual(['inbox']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('leaves a qualified ref unresolved when its schema misses, never another schema\'s table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['sales.orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"sales.orders" is referenced but was not found') + expect(migrations[0].sql).not.toContain('CREATE TABLE "') + }) + + it('emits all CREATE TABLEs before any FK constraint so circular FKs work', async () => { + const cyclicSchema = { + public: { + a: { + name: 'a', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'b_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.b', + columns: [{ source_column: 'b_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + }, + b: { + name: 'b', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'a_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.a', + columns: [{ source_column: 'a_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(cyclicSchema) + const usage = new Map([['main', new Set(['a', 'b'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('"public"."a"') + expect(sql).toContain('"public"."b"') + // Both FK constraints present, and every CREATE TABLE precedes the first one. + expect(sql.match(/ADD CONSTRAINT/g)?.length).toBe(2) + const lastCreate = sql.lastIndexOf('CREATE TABLE IF NOT EXISTS') + const firstConstraint = sql.indexOf('DO $$') + expect(lastCreate).toBeGreaterThan(-1) + expect(firstConstraint).toBeGreaterThan(lastCreate) + }) + + it('guards FK creation so re-running on an existing table does not abort', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + // The ADD CONSTRAINT must be wrapped in a pg_constraint existence check. + expect(sql).toContain('DO $$') + expect(sql).toContain('SELECT 1 FROM pg_constraint') + expect(sql).toContain(`conrelid = '"public"."orders"'::regclass`) + // No unguarded ALTER TABLE ... ADD at the start of a line. + expect(/^ALTER TABLE .* ADD CONSTRAINT/m.test(sql)).toBe(false) + }) + + it('creates non-public schemas before their tables', async () => { + const appSchema = { + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(appSchema) + const usage = new Map([['main', new Set(['app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS "app";') + expect(sql.indexOf('CREATE SCHEMA IF NOT EXISTS "app";')).toBeLessThan( + sql.indexOf('CREATE TABLE IF NOT EXISTS "app"."customers"') + ) + expect(sql).not.toContain('CREATE SCHEMA IF NOT EXISTS "public"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..b4e72389c0 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,345 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { allFlowModules } from './projectBundle' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateAddedTableSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of allFlowModules(item.value)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). A schema-qualified ref that +// misses stays unresolved: falling back to a same-named table in another +// schema would generate a migration for an unrelated table while the code +// still references the missing one. +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + return schema[schemaName]?.[tableName] ? { schemaName, tableName } : undefined + } + // Bare name: find it across every schema, first match wins. + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][tableRef]) return { schemaName, tableName: tableRef } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateAddedTableSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + // Every CREATE TABLE is emitted before any FK constraint: circular FKs have + // no valid creation order, so constraints can only run once all tables exist. + const creates: string[] = [] + const constraints: string[] = [] + for (const t of ordered) { + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + const gen = generateAddedTableSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + if (!gen) continue + creates.push(gen.create) + constraints.push(...gen.constraints) + } + const statements = [...creates, ...constraints] + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/lib/components/workspaceTree.test.ts b/frontend/src/lib/components/workspaceTree.test.ts index 9698da9f71..3e1dabd7a5 100644 --- a/frontend/src/lib/components/workspaceTree.test.ts +++ b/frontend/src/lib/components/workspaceTree.test.ts @@ -71,11 +71,11 @@ describe('buildWorkspaceTree', () => { kinds: ['flow'], loadingKind: {} }) - // At the top we should see the scope dirs (f/demo, u/alice) directly, + // At the top we should see the scope dirs (u/alice, f/demo) directly, // not a single 'kind:flow' branch wrapping them. expect(tree.every((n) => isBranch(n) && n.key.startsWith('dir:flow:'))).toBe(true) - // f-scopes come before u-scopes - expect(tree.map((n) => n.key)).toEqual([dirKey('flow', 'f/demo'), dirKey('flow', 'u/alice')]) + // u-scopes come before f-scopes + expect(tree.map((n) => n.key)).toEqual([dirKey('flow', 'u/alice'), dirKey('flow', 'f/demo')]) }) }) @@ -125,8 +125,8 @@ describe('buildWorkspaceTree', () => { kinds: ['flow'], loadingKind: {} }) - // Top-level: f/demo (folder scope), u/alice (user scope) - expect(tree.map((n) => n.key)).toEqual([dirKey('flow', 'f/demo'), dirKey('flow', 'u/alice')]) + // Top-level: u/alice (user scope), f/demo (folder scope) + expect(tree.map((n) => n.key)).toEqual([dirKey('flow', 'u/alice'), dirKey('flow', 'f/demo')]) const demo = findBranch(tree, dirKey('flow', 'f/demo')) // Children: nested folder `sub` first, then leaf `a` expect(childKeys(demo)).toEqual([ @@ -178,6 +178,41 @@ describe('buildWorkspaceTree', () => { expect(leaf.secondary).toBeUndefined() }) + it('labels and groups a draft-only item by its friendly draftPath, keyed by storage path', () => { + const draft = { ...item('app', 'u/admin/draft_abc123'), draftPath: 'f/marketing/dashboard' } + const tree = buildWorkspaceTree({ + loaded: { app: [draft] }, + kinds: ['app'], + loadingKind: {} + }) + // Grouped under the friendly folder, not u/admin. + expect(tree.map((n) => n.key)).toEqual([dirKey('app', 'f/marketing')]) + const marketing = findBranch(tree, dirKey('app', 'f/marketing')) + const leaf = marketing.children[0] + if (!isLeaf(leaf)) throw new Error('expected leaf') + // Displayed by the friendly path; keyed (and navigated) by storage path. + expect(leaf.label).toBe('f/marketing/dashboard') + expect(leaf.key).toBe(leafKeyFor('app', 'u/admin/draft_abc123')) + expect(leaf.data.path).toBe('u/admin/draft_abc123') + }) + + it('uses the friendly draftPath as secondary when a summary is present', () => { + const draft = { + ...item('script', 'u/admin/draft_xyz', 'My Script'), + draftPath: 'u/admin/my_script' + } + const tree = buildWorkspaceTree({ + loaded: { script: [draft] }, + kinds: ['script'], + loadingKind: {} + }) + const admin = findBranch(tree, dirKey('script', 'u/admin')) + const leaf = admin.children[0] + if (!isLeaf(leaf)) throw new Error('expected leaf') + expect(leaf.label).toBe('My Script') + expect(leaf.secondary).toBe('u/admin/my_script') + }) + it('marks the currentItem leaf with current=true', () => { const tree = buildWorkspaceTree({ loaded: { flow: [item('flow', 'f/demo/a'), item('flow', 'f/demo/b')] }, @@ -218,6 +253,45 @@ describe('buildWorkspaceTree', () => { expect(paths).not.toContain(leafKeyFor('flow', 'f/demo/old')) }) + it('does not duplicate a draft-only row whose friendly draftPath is the current live path', () => { + // Editor open on a renamed draft-only script: currentItem.path is the + // friendly path while listScripts returns the storage-path row carrying + // the same friendly path as draftPath. One leaf, marked current. + const loadedDraft = { + ...item('script', 'u/admin/draft_abc'), + draftPath: 'u/admin/my_script' + } + const tree = buildWorkspaceTree({ + loaded: { script: [loadedDraft] }, + kinds: ['script'], + loadingKind: {}, + currentItem: item('script', 'u/admin/my_script') + }) + const admin = findBranch(tree, dirKey('script', 'u/admin')) + expect(admin.children.map((c) => c.key)).toEqual([leafKeyFor('script', 'u/admin/draft_abc')]) + const leaf = admin.children[0] + if (!isLeaf(leaf)) throw new Error('expected leaf') + expect(leaf.current).toBe(true) + }) + + it('drops the storage-path row via draftPath during a mid-rename', () => { + // Renaming a draft-only item: savedPath is the old friendly path, which + // the loaded row only knows as its draftPath. The stale row must go so + // only the live (typed) entry shows. + const loadedDraft = { + ...item('script', 'u/admin/draft_abc'), + draftPath: 'u/admin/old_name' + } + const tree = buildWorkspaceTree({ + loaded: { script: [loadedDraft] }, + kinds: ['script'], + loadingKind: {}, + currentItem: { ...item('script', 'u/admin/new_name'), savedPath: 'u/admin/old_name' } + }) + const admin = findBranch(tree, dirKey('script', 'u/admin')) + expect(admin.children.map((c) => c.key)).toEqual([leafKeyFor('script', 'u/admin/new_name')]) + }) + it('does not re-inject when the live entry already exists in loaded', () => { const tree = buildWorkspaceTree({ loaded: { flow: [item('flow', 'f/demo/a', 'Original')] }, @@ -284,6 +358,46 @@ describe('buildWorkspaceTree', () => { expect(keys).toContain(leafKeyFor('script', 'f/demo/b')) }) + it('drops a live-cell extra at the friendly path when a loaded row carries it as draftPath', () => { + // listApps returns the draft-only row at its storage path with the + // friendly path in draftPath; the live editor cell surfaces the same + // draft as an extra keyed by the friendly path. One leaf, not two. + const loadedDraft = { ...item('app', 'u/admin/draft_abc'), draftPath: 'u/admin/dashboard' } + const tree = buildWorkspaceTree({ + loaded: { app: [loadedDraft] }, + kinds: ['app'], + loadingKind: {}, + extraItemsByKind: { app: [item('app', 'u/admin/dashboard')] } + }) + const admin = findBranch(tree, dirKey('app', 'u/admin')) + expect(admin.children.map((c) => c.key)).toEqual([leafKeyFor('app', 'u/admin/draft_abc')]) + }) + + it('folds a mid-rename live extra into the stale loaded row: one storage-keyed leaf under the typed folder', () => { + // Session picker while a rename's autosave is pending: listApps still + // carries the pre-rename friendly path, the live cell extra (re-keyed to + // the storage path by the picker) carries the typed one, and the tab's + // currentItem is the storage path. The typed name must win, on a single + // leaf that navigates via the storage path — never the display path. + const staleLoaded = { ...item('app', 'u/admin/draft_abc'), draftPath: 'u/admin/old_name' } + const liveExtra = { ...item('app', 'u/admin/draft_abc'), draftPath: 'f/marketing/new_name' } + const tree = buildWorkspaceTree({ + loaded: { app: [staleLoaded] }, + kinds: ['app'], + loadingKind: {}, + extraItemsByKind: { app: [liveExtra] }, + currentItem: item('app', 'u/admin/draft_abc') + }) + expect(tree.map((n) => n.key)).toEqual([dirKey('app', 'f/marketing')]) + const marketing = findBranch(tree, dirKey('app', 'f/marketing')) + expect(marketing.children.map((c) => c.key)).toEqual([leafKeyFor('app', 'u/admin/draft_abc')]) + const leaf = marketing.children[0] + if (!isLeaf(leaf)) throw new Error('expected leaf') + expect(leaf.current).toBe(true) + expect(leaf.data.path).toBe('u/admin/draft_abc') + expect(leaf.label).toBe('f/marketing/new_name') + }) + it('is a no-op when extras are absent or empty', () => { const noOpts = buildWorkspaceTree({ loaded: { flow: [item('flow', 'f/demo/a')] }, @@ -299,6 +413,57 @@ describe('buildWorkspaceTree', () => { expect(JSON.stringify(noOpts)).toEqual(JSON.stringify(emptyExtras)) }) }) + + describe('flat layout', () => { + it('roots the cross-kind scope dirs directly (no All / kind branches)', () => { + const tree = buildWorkspaceTree({ + loaded: { + flow: [item('flow', 'f/demo/a'), item('flow', 'u/alice/b')], + script: [item('script', 'f/demo/c')] + }, + kinds: ['flow', 'script'], + loadingKind: {}, + layout: 'flat' + }) + // u-scopes before f-scopes, keyed under the 'all' namespace. + expect(tree.map((n) => n.key)).toEqual([dirKey('all', 'u/alice'), dirKey('all', 'f/demo')]) + }) + + it('mixes every kind inside the same scope dir', () => { + const tree = buildWorkspaceTree({ + loaded: { + flow: [item('flow', 'f/demo/a')], + script: [item('script', 'f/demo/b')] + }, + kinds: ['flow', 'script'], + loadingKind: {}, + layout: 'flat' + }) + const demo = findBranch(tree, dirKey('all', 'f/demo')) + expect(childKeys(demo)).toEqual([ + leafKeyFor('flow', 'f/demo/a'), + leafKeyFor('script', 'f/demo/b') + ]) + }) + + it('applies extras and currentItem like the by-kind layout', () => { + const tree = buildWorkspaceTree({ + loaded: { flow: [item('flow', 'f/demo/a')], script: [] }, + kinds: ['flow', 'script'], + loadingKind: {}, + extraItemsByKind: { script: [item('script', 'f/demo/draft')] }, + currentItem: item('flow', 'f/demo/a'), + layout: 'flat' + }) + const demo = findBranch(tree, dirKey('all', 'f/demo')) + const leaves = demo.children.filter(isLeaf) + expect(leaves.map((l) => l.key)).toEqual([ + leafKeyFor('flow', 'f/demo/a'), + leafKeyFor('script', 'f/demo/draft') + ]) + expect(leaves[0].current).toBe(true) + }) + }) }) describe('legacyScopeToPath', () => { @@ -333,6 +498,16 @@ describe('legacyScopeToPath', () => { dirKey('flow', 'f/demo') ]) }) + + it('flat: returns [dirKey under all] for a dir scope, ignoring the kind', () => { + expect(legacyScopeToPath({ kind: 'flow', dir: 'f/demo' }, ['flow', 'script'], 'flat')).toEqual([ + dirKey('all', 'f/demo') + ]) + }) + + it('flat: returns [] without a dir', () => { + expect(legacyScopeToPath({ kind: 'all' }, ['flow', 'script'], 'flat')).toEqual([]) + }) }) describe('relativizeWorkspacePath', () => { diff --git a/frontend/src/lib/components/workspaceTree.ts b/frontend/src/lib/components/workspaceTree.ts index e7da20b62d..3a922ef45b 100644 --- a/frontend/src/lib/components/workspaceTree.ts +++ b/frontend/src/lib/components/workspaceTree.ts @@ -4,6 +4,7 @@ import { KIND_LABEL, kindKey, leafKeyFor, + workspaceItemDisplayPath, type WorkspaceItem, type WorkspaceItemKind } from './workspacePicker' @@ -21,11 +22,13 @@ type DirNode = { leaves: WorkspaceItem[] } -/** Build the path-hierarchy from a flat list of workspace items. */ +/** Build the path-hierarchy from a flat list of workspace items. Items are + * placed by their display path, so a draft-only item shows up under its + * friendly folder rather than the `u//draft_` storage location. */ function buildDirForest(items: WorkspaceItem[]): DirNode[] { const scopeRoots = new Map() for (const it of items) { - const parts = it.path.split('/') + const parts = workspaceItemDisplayPath(it).split('/') if (parts.length < 3) continue const scopeFp = parts.slice(0, 2).join('/') let node = scopeRoots.get(scopeFp) @@ -48,21 +51,29 @@ function buildDirForest(items: WorkspaceItem[]): DirNode[] { cur.leaves.push(it) } const scopes = Array.from(scopeRoots.values()).sort((a, b) => { - // `f/` (folder) scopes before `u/` (user) scopes; alphabetical within. - const af = a.fullPath.startsWith('f/') ? 0 : 1 - const bf = b.fullPath.startsWith('f/') ? 0 : 1 - if (af !== bf) return af - bf + // `u/` (user) scopes before `f/` (folder) scopes; alphabetical within. + const au = a.fullPath.startsWith('u/') ? 0 : 1 + const bu = b.fullPath.startsWith('u/') ? 0 : 1 + if (au !== bu) return au - bu return a.fullPath.localeCompare(b.fullPath) }) const sortNode = (n: DirNode) => { n.children.sort((a, b) => a.name.localeCompare(b.name)) - n.leaves.sort((a, b) => a.path.localeCompare(b.path)) + n.leaves.sort((a, b) => workspaceItemDisplayPath(a).localeCompare(workspaceItemDisplayPath(b))) n.children.forEach(sortNode) } scopes.forEach(sortNode) return scopes } +/** True when `p` names this item — its storage path or its friendly draft + * path. A draft-only editor's live/saved paths are the friendly path while + * the loaded row sits at the storage path, so matching on `path` alone would + * treat them as two different items. */ +function itemMatchesPath(it: WorkspaceItem, p: string | undefined): boolean { + return p !== undefined && (it.path === p || it.draftPath === p) +} + /** Inject the currently-edited item at its live path, dropping the saved * entry when a draft rename is mid-flight. Only applies to items of the * same kind. */ @@ -74,9 +85,9 @@ function withCurrent( if (!currentItem || currentItem.kind !== k) return items const drafted = currentItem.savedPath && currentItem.savedPath !== currentItem.path - ? items.filter((it) => it.path !== currentItem.savedPath) + ? items.filter((it) => !itemMatchesPath(it, currentItem.savedPath)) : items - if (drafted.some((it) => it.path === currentItem.path)) return drafted + if (drafted.some((it) => itemMatchesPath(it, currentItem.path))) return drafted return [ ...drafted, { @@ -92,12 +103,14 @@ function itemToLeaf( it: WorkspaceItem, currentItem: (WorkspaceItem & { savedPath?: string }) | undefined ): DrillLeaf { - const isCurrent = !!currentItem && currentItem.kind === it.kind && currentItem.path === it.path + const isCurrent = + !!currentItem && currentItem.kind === it.kind && itemMatchesPath(it, currentItem.path) + const display = workspaceItemDisplayPath(it) return { type: 'leaf', key: leafKeyFor(it.kind, it.path), - label: it.summary || it.path, - secondary: it.summary ? it.path : undefined, + label: it.summary || display, + secondary: it.summary ? display : undefined, data: it, current: isCurrent } @@ -126,8 +139,10 @@ function dirToBranch( /** Merge AI-created in-memory drafts (or any caller-provided extras) into a * kind's loaded list. The chat tools / session previews scaffold items via * `UserDraft` before the user deploys; those should be navigable from the - * picker. Existing items (same path) win so backend metadata (summary etc.) - * isn't clobbered. */ + * picker. An extra matching a loaded item (by storage or friendly path — else + * one draft renders as two leaves) is folded into it: the loaded row wins on + * backend metadata (summary etc.), but the extra's `draftPath` is overlaid + * when set — a live editor cell knows a rename before the backend list does. */ function withExtras( items: WorkspaceItem[], k: WorkspaceItemKind, @@ -135,12 +150,21 @@ function withExtras( ): WorkspaceItem[] { const extras = extraItemsByKind?.[k] if (!extras || extras.length === 0) return items - const known = new Set(items.map((it) => it.path)) - return items.concat(extras.filter((d) => !known.has(d.path))) + const leftover = new Set(extras) + const merged = items.map((it) => { + const ex = extras.find((d) => itemMatchesPath(it, d.path) || itemMatchesPath(it, d.draftPath)) + if (!ex) return it + leftover.delete(ex) + return ex.draftPath !== undefined && ex.draftPath !== it.draftPath + ? { ...it, draftPath: ex.draftPath } + : it + }) + return leftover.size > 0 ? merged.concat([...leftover]) : merged } /** Build the workspace drill tree. * + * Default `by-kind` layout: * - One branch per kind in `kinds` (`Flows` / `Scripts` / `Apps`), * each containing the kind's path hierarchy. * - When `kinds.length > 1`, prepend an `All` branch that merges items @@ -148,6 +172,12 @@ function withExtras( * leaves don't appear twice in global-search results. * - When `kinds.length === 1`, return the single kind branch's children * directly so the user lands on folders without a redundant level. + * + * `flat` layout: + * - No kind grouping — the root IS the workspace home's first level: the + * `f/` / `u/` scope dirs of the cross-kind merge, mixing + * every kind's items inside. Callers must eager-load all kinds (there is + * no per-kind drill step left to lazy-load from). */ export function buildWorkspaceTree(opts: { loaded: Partial> @@ -160,9 +190,11 @@ export function buildWorkspaceTree(opts: { * (e.g. AI-created localStorage drafts surfaced by the workspace adapter). * Extras whose path matches an already-loaded item are dropped. */ extraItemsByKind?: Partial> + layout?: 'by-kind' | 'flat' }): DrillNode[] { const { loaded, kinds, currentItem, extraItemsByKind } = opts const loadingKind = opts.loadingKind ?? {} + const layout = opts.layout ?? 'by-kind' function kindBranch(k: WorkspaceItemKind): DrillBranch { const raw = withExtras(loaded[k] ?? [], k, extraItemsByKind) @@ -182,14 +214,23 @@ export function buildWorkspaceTree(opts: { if (kinds.length === 0) return [] + const mergedItems = () => + kinds.flatMap((k) => + withCurrent(withExtras(loaded[k] ?? [], k, extraItemsByKind), k, currentItem) + ) + + if (layout === 'flat') { + const items = mergedItems() + const dirs = items.length > 0 ? buildDirForest(items) : [] + return dirs.map((d) => dirToBranch(d, 'all', currentItem)) + } + if (kinds.length === 1) { return kindBranch(kinds[0]).children } // Cross-kind 'all' branch — flagged so search doesn't double-count leaves. - const allItems = kinds.flatMap((k) => - withCurrent(withExtras(loaded[k] ?? [], k, extraItemsByKind), k, currentItem) - ) + const allItems = mergedItems() const allDirs = allItems.length > 0 ? buildDirForest(allItems) : [] const allBranch: DrillBranch = { type: 'branch', @@ -208,9 +249,15 @@ export function buildWorkspaceTree(opts: { * (BreadcrumbSegment / EditorHeader) onto the new generic `string[]` path. */ export function legacyScopeToPath( scope: { kind: WorkspaceItemKind | 'all'; dir?: string } | undefined, - kinds: WorkspaceItemKind[] + kinds: WorkspaceItemKind[], + layout: 'by-kind' | 'flat' = 'by-kind' ): string[] { if (!scope) return [] + // Flat layout: no kind branches at all — scope dirs live at root and are + // always keyed under the cross-kind 'all' namespace. + if (layout === 'flat') { + return scope.dir ? [dirKey('all', scope.dir)] : [] + } // Single-kind mode: there's no kind branch at root; scope's `kind` is // implicit. Only the dir (if any) makes it to the path. if (kinds.length === 1) { diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index 95afa84802..11d801543c 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -22,8 +22,6 @@ export const SIDEBAR_SHOW_SCHEDULES = true export const WORKSPACE_SHOW_SLACK_CMD = true export const WORKSPACE_SHOW_WEBHOOK_CLI_SYNC = true -export const SCRIPT_VIEW_SHOW_PUBLISH_TO_HUB = true - export const SCRIPT_VIEW_SHOW_SCHEDULE = true export const SCRIPT_VIEW_SHOW_EXAMPLE_CURL = true diff --git a/frontend/src/lib/forkParentMemory.test.ts b/frontend/src/lib/forkParentMemory.test.ts new file mode 100644 index 0000000000..48d0d553fb --- /dev/null +++ b/frontend/src/lib/forkParentMemory.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import type { UserWorkspace } from './stores' +import { recordForkParent, getRememberedForkParent, forgetForkParent } from './forkParentMemory' + +const MAX_ENTRIES = 50 + +function ws(id: string, parent_workspace_id?: string): UserWorkspace { + return { + id, + name: id, + username: 'admin', + color: undefined, + operator_settings: undefined, + disabled: false, + parent_workspace_id + } +} + +beforeEach(() => { + localStorage.clear() +}) + +describe('recordForkParent / getRememberedForkParent', () => { + it('remembers the parent of a fork present in the list', () => { + recordForkParent('wm-fork-a', [ws('wm-fork-a', 'parent'), ws('parent')]) + expect(getRememberedForkParent('wm-fork-a')).toBe('parent') + }) + + it('is a no-op when the workspace is absent from the list', () => { + recordForkParent('wm-fork-a', [ws('parent')]) + expect(getRememberedForkParent('wm-fork-a')).toBeUndefined() + }) + + it('is a no-op when the workspace has no parent', () => { + recordForkParent('parent', [ws('parent')]) + expect(getRememberedForkParent('parent')).toBeUndefined() + }) + + it('is a no-op for an undefined workspace id', () => { + recordForkParent(undefined, [ws('wm-fork-a', 'parent')]) + expect(getRememberedForkParent('wm-fork-a')).toBeUndefined() + }) + + it('never clobbers a prior mapping once the fork disappears from the list', () => { + // Recorded while reachable... + recordForkParent('wm-fork-a', [ws('wm-fork-a', 'parent'), ws('parent')]) + // ...then the fork is gone from a later list — the mapping must survive so + // recovery can still find the parent. + recordForkParent('wm-fork-a', [ws('parent')]) + expect(getRememberedForkParent('wm-fork-a')).toBe('parent') + }) + + it('handles workspace ids that collide with Object prototype members', () => { + for (const id of ['__proto__', 'constructor', 'toString']) { + recordForkParent(id, [ws(id, `parent-${id}`), ws(`parent-${id}`)]) + expect(getRememberedForkParent(id)).toBe(`parent-${id}`) + forgetForkParent(id) + expect(getRememberedForkParent(id)).toBeUndefined() + } + }) +}) + +describe('forgetForkParent', () => { + it('removes a remembered mapping', () => { + recordForkParent('wm-fork-a', [ws('wm-fork-a', 'parent')]) + forgetForkParent('wm-fork-a') + expect(getRememberedForkParent('wm-fork-a')).toBeUndefined() + }) + + it('is a no-op for an unknown fork', () => { + expect(() => forgetForkParent('wm-fork-missing')).not.toThrow() + }) +}) + +describe('bounded storage', () => { + it('evicts the oldest entry once past MAX_ENTRIES', () => { + for (let i = 0; i < MAX_ENTRIES; i++) { + recordForkParent(`wm-fork-${i}`, [ws(`wm-fork-${i}`, `parent-${i}`)]) + } + // One more tips it over the cap and trims the oldest (index 0). + recordForkParent(`wm-fork-${MAX_ENTRIES}`, [ + ws(`wm-fork-${MAX_ENTRIES}`, `parent-${MAX_ENTRIES}`) + ]) + expect(getRememberedForkParent('wm-fork-0')).toBeUndefined() + expect(getRememberedForkParent(`wm-fork-${MAX_ENTRIES}`)).toBe(`parent-${MAX_ENTRIES}`) + expect(getRememberedForkParent('wm-fork-1')).toBe('parent-1') + }) + + it('updating a fork to a new parent moves it to the end so it survives the next eviction', () => { + for (let i = 0; i < MAX_ENTRIES; i++) { + recordForkParent(`wm-fork-${i}`, [ws(`wm-fork-${i}`, `parent-${i}`)]) + } + // A changed value re-inserts the key at the end of the insertion order. + recordForkParent('wm-fork-0', [ws('wm-fork-0', 'reparented')]) + // Push past the cap: the now-oldest (index 1) is evicted, not index 0. + recordForkParent(`wm-fork-${MAX_ENTRIES}`, [ + ws(`wm-fork-${MAX_ENTRIES}`, `parent-${MAX_ENTRIES}`) + ]) + expect(getRememberedForkParent('wm-fork-0')).toBe('reparented') + expect(getRememberedForkParent('wm-fork-1')).toBeUndefined() + }) +}) + +describe('corrupted storage', () => { + it('tolerates non-JSON content', () => { + localStorage.setItem('fork_parents', 'not json{') + expect(getRememberedForkParent('wm-fork-a')).toBeUndefined() + // A subsequent write recovers cleanly. + recordForkParent('wm-fork-a', [ws('wm-fork-a', 'parent')]) + expect(getRememberedForkParent('wm-fork-a')).toBe('parent') + }) + + it('tolerates a non-object JSON value', () => { + localStorage.setItem('fork_parents', '42') + expect(getRememberedForkParent('wm-fork-a')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/forkParentMemory.ts b/frontend/src/lib/forkParentMemory.ts new file mode 100644 index 0000000000..8cb2a3c4e0 --- /dev/null +++ b/frontend/src/lib/forkParentMemory.ts @@ -0,0 +1,82 @@ +import type { UserWorkspace } from './stores' + +// A fork's parent linkage lives only in its own `workspace` row. Once the fork +// is deleted remotely, `listUserWorkspaces` stops returning it and the parent is +// unrecoverable from the server. We therefore mirror `fork id -> parent id` into +// localStorage while the fork is still reachable, so that after a reload landing +// on a now-deleted fork we can send the user back to its parent instead of a +// dead workspace / forced logout. + +const FORK_PARENTS_KEY = 'fork_parents' +const MAX_ENTRIES = 50 + +// A null-prototype map so workspace ids that collide with Object prototype members +// (`__proto__`, `constructor`, …) are stored as plain own properties rather than +// triggering prototype semantics on read/write. +function emptyMap(): Record { + return Object.create(null) +} + +function readMap(): Record { + const map = emptyMap() + try { + const raw = localStorage.getItem(FORK_PARENTS_KEY) + if (!raw) return map + const parsed = JSON.parse(raw) + if (!parsed || typeof parsed !== 'object') return map + for (const key of Object.keys(parsed)) { + const value = (parsed as Record)[key] + if (typeof value === 'string') map[key] = value + } + } catch (e) { + console.error('Could not read fork parent mapping', e) + } + return map +} + +function writeMap(map: Record): void { + try { + localStorage.setItem(FORK_PARENTS_KEY, JSON.stringify(map)) + } catch (e) { + console.error('Could not persist fork parent mapping', e) + } +} + +export function rememberForkParent(forkId: string, parentId: string): void { + const map = readMap() + if (map[forkId] === parentId) return + // Re-insert at the end so the oldest entries are the ones trimmed below. + delete map[forkId] + map[forkId] = parentId + const keys = Object.keys(map) + if (keys.length > MAX_ENTRIES) { + for (const k of keys.slice(0, keys.length - MAX_ENTRIES)) delete map[k] + } + writeMap(map) +} + +export function getRememberedForkParent(forkId: string): string | undefined { + return readMap()[forkId] +} + +export function forgetForkParent(forkId: string): void { + const map = readMap() + if (forkId in map) { + delete map[forkId] + writeMap(map) + } +} + +// Records the parent of the current workspace when it is a fork still present in +// the user's workspace list. No-op otherwise, so it never clobbers a previously +// remembered parent when the fork has already disappeared. +export function recordForkParent( + workspaceId: string | undefined, + workspaces: UserWorkspace[] +): void { + if (!workspaceId) return + const ws = workspaces.find((w) => w.id === workspaceId) + if (ws?.parent_workspace_id) { + rememberForkParent(workspaceId, ws.parent_workspace_id) + } +} diff --git a/frontend/src/lib/hub.ts b/frontend/src/lib/hub.ts index a038f18580..25b3f8b3fd 100644 --- a/frontend/src/lib/hub.ts +++ b/frontend/src/lib/hub.ts @@ -1,6 +1,4 @@ -import type { Schema } from './common' -import { AppService, FlowService, type Flow, type Script } from './gen' -import { encodeState } from './utils' +import { AppService, FlowService } from './gen' import hubPathsData from './hubPaths.json' import { replacePlaceholderForSignatureScriptTemplate, @@ -11,22 +9,6 @@ import { export const DEFAULT_HUB_BASE_URL = 'https://hub.windmill.dev' export const PRIVATE_HUB_MIN_VERSION = 10_000_000 -export function scriptToHubUrl( - content: string, - summary: string, - description: string, - kind: Script['kind'], - language: Script['language'], - schema: Schema | any, - lock: string | undefined, - hubBaseUrl: string -): URL { - const url = new URL(hubBaseUrl + '/scripts/add') - url.hash = encodeState({ content, summary, description, kind, language, schema, lock }) - - return url -} - export const HubScript = { SIGNATURE_TEMPLATE: SIGNATURE_TEMPLATE_SCRIPT_HUB_PATH } as const @@ -65,32 +47,6 @@ export async function loadHubApps() { } } -export function flowToHubUrl(flow: Flow, hubBaseUrl: string): URL { - const url = new URL(hubBaseUrl + '/flows/add') - const openFlow = { - value: flow.value, - summary: flow.summary, - description: flow.description, - schema: flow.schema - } - url.searchParams.append('flow', encodeState(openFlow)) - return url -} - -export function appToHubUrl(staticApp: any, hubBaseUrl: string): URL { - const url = new URL(hubBaseUrl + '/apps/add') - url.searchParams.append('app', encodeState(staticApp)) - return url -} - -export function rawAppToHubUrl(hubBaseUrl: string, summary?: string): URL { - const url = new URL(hubBaseUrl + '/raw_apps/add') - if (summary) { - url.searchParams.append('summary', summary) - } - return url -} - type HubPaths = { gitSyncTest: string gitInitRepo: string diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2ae15d931b..3a9874596d 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,14 +1,14 @@ { "gitSyncTest": "hub/28184/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28229/git-sync%3A-init-repository-windmill", - "slackErrorHandler": "hub/28241/workspace-or-schedule-error-handler-slack", + "gitInitRepo": "hub/28808/git-sync-init-repository-windmill", + "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", - "slackRecoveryHandler": "hub/28239/slack/schedule-recovery-handler-slack", - "slackSuccessHandler": "hub/28240/slack/schedule-success-handler-slack", + "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", + "slackSuccessHandler": "hub/28793/slack/schedule-success-handler-slack", "teamsErrorHandler": "hub/19742/workspace-or-schedule-error-handler-teams", "teamsRecoveryHandler": "hub/11593/schedule-recovery-handler-teams", "teamsSuccessHandler": "hub/11596/schedule-success-handler-teams", - "slackReport": "hub/9084/slack", + "slackReport": "hub/28792/slack", "discordReport": "hub/9085/discord", "smtpReport": "hub/28242/smtp", "appReport": "hub/28243/app-report", diff --git a/frontend/src/lib/isChromiumBrowser.test.ts b/frontend/src/lib/isChromiumBrowser.test.ts new file mode 100644 index 0000000000..59e325e407 --- /dev/null +++ b/frontend/src/lib/isChromiumBrowser.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { isChromiumBrowser } from './utils' + +// Gates Chromium-only capabilities (DOM screenshot capture). A wrong "true" on +// Gecko/WebKit re-enables the capture path whose spacing/wrapping artifacts the +// gate exists to avoid; a wrong "false" on Chromium silently drops the tool. +describe('isChromiumBrowser', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('detects Chromium via userAgentData brands (any Chromium-based browser)', () => { + vi.stubGlobal('navigator', { + userAgentData: { + brands: [ + { brand: 'Not-A.Brand', version: '99' }, + { brand: 'Chromium', version: '138' }, + { brand: 'Microsoft Edge', version: '138' } + ] + }, + userAgent: 'anything' + }) + expect(isChromiumBrowser()).toBe(true) + }) + + it('falls back to the UA string when userAgentData is absent (older Chromium)', () => { + vi.stubGlobal('navigator', { + userAgent: + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36' + }) + expect(isChromiumBrowser()).toBe(true) + }) + + it('is false on Firefox ("Chrome/" never appears in a Gecko UA)', () => { + vi.stubGlobal('navigator', { + userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0' + }) + expect(isChromiumBrowser()).toBe(false) + }) + + it('is false with no navigator at all (SSR)', () => { + vi.stubGlobal('navigator', undefined) + expect(isChromiumBrowser()).toBe(false) + }) +}) diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index 8a46163009..70dabb01f6 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -10,7 +10,6 @@ export interface EndpointTool { pathParamsSchema?: object; queryParamsSchema?: object; bodySchema?: object; - pathFieldRenames?: Record; queryFieldRenames?: Record; bodyFieldRenames?: Record; } @@ -36,7 +35,6 @@ export const mcpEndpointTools: EndpointTool[] = [ ] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -64,7 +62,48 @@ export const mcpEndpointTools: EndpointTool[] = [ ] }, bodySchema: undefined, - pathFieldRenames: undefined, + queryFieldRenames: undefined, + bodyFieldRenames: undefined + }, + { + name: "listDataMetrics", + description: "list declared measures and dimensions on DuckLake tables: Call this before writing any aggregate query over a DuckLake table. A declared measure is the canonical definition of that number, and reproducing it yourself will silently disagree with it (a `revenue` measure typically excludes refunds or test rows). Filter by `table` for one table's declarations, or by `path_prefix` (e.g. `f/analytics`) for everything declared under a folder; omit both to browse the whole catalog. Results are keyset-paged: a full page may mean more remain, so continue with the `cursor_*` params rather than assuming a measure does not exist. Use each returned `expr` verbatim, and when a measure has a `filter` write it as `expr FILTER (WHERE filter)` so measures with different predicates can share one GROUP BY. If a number you need has no declared measure, write your own aggregate as usual. Results are limited to declarations whose producing script the caller can read", + instructions: "", + path: "/w/{workspace}/data_metrics/list", + method: "GET", + pathParamsSchema: undefined, + queryParamsSchema: { + "type": "object", + "properties": { + "table": { + "type": "string", + "description": "DuckLake table path, with or without the `ducklake://` scheme" + }, + "path_prefix": { + "type": "string", + "description": "Producing script path prefix, e.g. `f/analytics`" + }, + "per_page": { + "type": "integer", + "description": "Results per page, capped at 1000 (default 1000)" + }, + "cursor_table": { + "type": "string", + "description": "Keyset cursor. To page, pass the previous response's `next_cursor` fields back as `cursor_*`; all four move together, and are omitted for the first page. Continue whenever `next_cursor` is present. Every returned row is one the caller may read, so the cursor never names a hidden row.\n" + }, + "cursor_kind": { + "type": "string" + }, + "cursor_name": { + "type": "string" + }, + "cursor_script": { + "type": "string" + } + }, + "required": [] +}, + bodySchema: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -134,7 +173,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -157,7 +195,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -170,13 +207,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: { @@ -215,12 +251,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "The path to the variable (body parameter)" + "description": "The path to the variable (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -263,7 +296,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -317,7 +349,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -372,7 +403,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "resource_type" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -395,7 +425,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -408,13 +437,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -443,12 +471,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "The path to the resource (body parameter)" + "description": "The path to the resource (body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -483,7 +508,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -545,7 +569,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -558,7 +581,6 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: undefined, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -656,7 +678,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -708,7 +729,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "language" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -731,7 +751,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -763,7 +782,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -798,7 +816,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -825,7 +842,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "The arguments to pass to the script or flow", "additionalProperties": true }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -895,7 +911,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -930,7 +945,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -976,7 +990,6 @@ export const mcpEndpointTools: EndpointTool[] = [ ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -989,13 +1002,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -1024,18 +1036,14 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } }, "required": [ "summary", - "value", - "path__body" + "value" ], "description": "Top-level flow definition containing metadata, configuration, and the flow structure" -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -1070,7 +1078,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1108,7 +1115,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "policy" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1121,13 +1127,12 @@ export const mcpEndpointTools: EndpointTool[] = [ pathParamsSchema: { "type": "object", "properties": { - "path__path": { - "type": "string", - "description": "(path parameter)" + "path": { + "type": "string" } }, "required": [ - "path__path" + "path" ] }, queryParamsSchema: undefined, @@ -1148,12 +1153,9 @@ export const mcpEndpointTools: EndpointTool[] = [ }, "path__body": { "type": "string", - "description": "(body parameter)" + "description": "(body parameter). Defaults to `path` when omitted; set it only to change the path." } } -}, - pathFieldRenames: { - "path__path": "path" }, queryFieldRenames: undefined, bodyFieldRenames: { @@ -1183,7 +1185,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "The arguments to pass to the script or flow", "additionalProperties": true }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1278,7 +1279,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "language" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1399,7 +1399,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1566,7 +1565,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1605,7 +1603,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1637,7 +1634,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -1847,7 +1843,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "args" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2050,7 +2045,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "args" ] }, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2073,7 +2067,6 @@ export const mcpEndpointTools: EndpointTool[] = [ }, queryParamsSchema: undefined, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2105,7 +2098,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2171,7 +2163,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined }, @@ -2201,7 +2192,6 @@ export const mcpEndpointTools: EndpointTool[] = [ "required": [] }, bodySchema: undefined, - pathFieldRenames: undefined, queryFieldRenames: undefined, bodyFieldRenames: undefined } diff --git a/frontend/src/lib/newDraftFlag.test.ts b/frontend/src/lib/newDraftFlag.test.ts index 95063aa34c..4a2141817e 100644 --- a/frontend/src/lib/newDraftFlag.test.ts +++ b/frontend/src/lib/newDraftFlag.test.ts @@ -17,6 +17,18 @@ vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn(), getLocalDraftHint: () => hints.value })) +// `stripNewDraftFlag` rewrites the URL through SvelteKit's `replaceState` and +// refreshes the session-switch's remembered nav route. Mock those so the strip +// is observable via `window.location.href` (mirroring jsdom) and the remembered +// route can be asserted. +const rememberNavRoute = vi.hoisted(() => vi.fn()) +vi.mock('$app/navigation', () => ({ + replaceState: (url: URL | string, _state: unknown) => { + window.location.href = new URL(url, window.location.href).toString() + } +})) +vi.mock('$app/state', () => ({ page: { state: {} } })) +vi.mock('$lib/components/sessions/sessionSwitch.svelte', () => ({ rememberNavRoute })) import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' import { stripNewDraftFlagOnSave, shouldSeedNewDraft } from './newDraftFlag' @@ -108,6 +120,9 @@ describe('stripNewDraftFlagOnSave', () => { expect(window.location.href).not.toContain('new_draft') // Sibling seeding params are preserved. expect(window.location.href).toContain('template=foo') + // The remembered nav route is refreshed to the stripped URL so exiting an + // AI session returns here without re-adding ?new_draft. + expect(rememberNavRoute).toHaveBeenCalledWith('/scripts/edit/u/me/draft_d?template=foo') }) it('does not strip on a delete save', async () => { diff --git a/frontend/src/lib/newDraftFlag.ts b/frontend/src/lib/newDraftFlag.ts index 20f45cf0e9..88e58f030a 100644 --- a/frontend/src/lib/newDraftFlag.ts +++ b/frontend/src/lib/newDraftFlag.ts @@ -1,16 +1,27 @@ +import { page } from '$app/state' +import { replaceState } from '$app/navigation' +import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { UserDraftDbSyncer, type UserDraftLastSyncQuery } from '$lib/userDraftDbSyncer.svelte' import { getLocalDraftHint } from '$lib/localDraftHints.svelte' import type { UserDraftItemKind } from '$lib/gen' /** Drop `?new_draft=true` from the current URL (preserving every other param), * mutating the address bar without a navigation. No-op when the flag is absent - * or `window` is unavailable (SSR). */ + * or `window` is unavailable (SSR). + * + * Uses SvelteKit's `replaceState` (not raw `history.replaceState`, which the + * router warns conflicts with it) so the history entry keeps the router's + * bookkeeping and `page.state`. Also refreshes the remembered nav route: + * `afterNavigate` never observes this in-place rewrite, so without it + * `exitSessionMode` would restore the pre-strip URL — still carrying + * `?new_draft=true` — and re-enter the seed-empty branch. */ export function stripNewDraftFlag(): void { if (typeof window === 'undefined') return const url = new URL(window.location.href) if (url.searchParams.get('new_draft') !== 'true') return url.searchParams.delete('new_draft') - window.history.replaceState(window.history.state, '', url.toString()) + replaceState(url, page.state) + rememberNavRoute(url.pathname + url.search) } /** diff --git a/frontend/src/lib/script_helpers.test.ts b/frontend/src/lib/script_helpers.test.ts new file mode 100644 index 0000000000..1badf03e11 --- /dev/null +++ b/frontend/src/lib/script_helpers.test.ts @@ -0,0 +1,43 @@ +import { describe, it, expect } from 'vitest' +import { bashRunsInCustomImage } from './script_helpers' + +// bashRunsInCustomImage decides whether the +Variable/+Resource pickers insert a +// curl/wget snippet (custom image, no wmill CLI) or the wmill CLI snippet. It must +// stay in sync with the worker's BashAnnotations grammar +// (backend/windmill-common/src/worker.rs): leading comment lines only, `# sandbox +// ` or bare `# docker` select a container; a bare `# sandbox` does not. + +describe('bashRunsInCustomImage', () => { + it('true for `# sandbox ` (spaced and compact)', () => { + expect(bashRunsInCustomImage('# sandbox alpine:latest\necho hi')).toBe(true) + expect(bashRunsInCustomImage('#sandbox python:3.12-slim\n')).toBe(true) + }) + + it('true for a bare `# docker` annotation', () => { + expect(bashRunsInCustomImage('# docker\necho hi')).toBe(true) + }) + + it('true when the sandbox line follows other leading comments (default template)', () => { + expect(bashRunsInCustomImage('# shellcheck shell=bash\n# sandbox alpine:latest\necho hi')).toBe( + true + ) + }) + + it('false for a bare `# sandbox` (nsjail-bash on the worker, wmill available)', () => { + expect(bashRunsInCustomImage('# sandbox\necho hi')).toBe(false) + }) + + it('false for prose comments that merely contain the words', () => { + expect(bashRunsInCustomImage('# sandboxed run below\necho hi')).toBe(false) + expect(bashRunsInCustomImage('# runs in a docker container\necho hi')).toBe(false) + }) + + it('false when the annotation is not on a leading comment line', () => { + expect(bashRunsInCustomImage('echo hi\n# sandbox alpine')).toBe(false) + expect(bashRunsInCustomImage('msg="$1" # docker')).toBe(false) + }) + + it('false for a plain script with no annotations', () => { + expect(bashRunsInCustomImage('# shellcheck shell=bash\necho hi')).toBe(false) + }) +}) diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index d936d60f25..91e310f5c9 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -703,7 +703,7 @@ export const TS_PREPROCESSOR_SCRIPT_INTRO = `/** * * ⚠️ This function runs BEFORE the main function. * - * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) + * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) * before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated runnable UI clean. * * The returned object defines the parameter values passed to \`main()\`. @@ -716,7 +716,7 @@ export const TS_PREPROCESSOR_SCRIPT_INTRO = `/** export const TS_PREPROCESSOR_FLOW_INTRO = `/** * Trigger preprocessor * - * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) + * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) * before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. * * The returned object determines the parameter values passed to the flow. @@ -812,6 +812,16 @@ type TriggerEvent = content_type?: string; }; } + | { + kind: "amqp"; + trigger_path: string; + payload: string; + exchange: string; + routing_key: string; + queue_name: string; + redelivered: boolean; + delivery_tag: number; + } | { kind: "gcp"; trigger_path: string; @@ -867,7 +877,7 @@ export const PYTHON_PREPROCESSOR_SCRIPT_INTRO = `# Trigger preprocessor # # ⚠️ This function runs BEFORE the main function. # -# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) +# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) # before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated UI clean. # # The returned object defines the parameter values passed to \`main()\`. @@ -878,7 +888,7 @@ export const PYTHON_PREPROCESSOR_SCRIPT_INTRO = `# Trigger preprocessor export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor # -# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) +# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) # before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. # # The returned object determines the parameter values passed to the flow. @@ -986,6 +996,17 @@ class MqttEvent(TypedDict): v5: Optional[MqttV5Properties] +class AmqpEvent(TypedDict): + kind: Literal["amqp"] + trigger_path: str + payload: str + exchange: str + routing_key: str + queue_name: str + redelivered: bool + delivery_tag: int + + class GcpEvent(TypedDict): kind: Literal["gcp"] trigger_path: str @@ -1019,6 +1040,7 @@ Event = Union[ NatsEvent, SqsEvent, MqttEvent, + AmqpEvent, GcpEvent, PostgresEvent, ] @@ -1036,7 +1058,7 @@ export const PHP_PREPROCESSOR_SCRIPT_INTRO = ` 'mqtt', 'trigger_path' => '...', 'payload' => '...', 'topic' => '...', 'retain' => true, 'pkid' => 1, // 'qos' => 1, 'v5' => [...]] // + // AMQP event: + // ['kind' => 'amqp', 'trigger_path' => '...', 'payload' => '...', 'exchange' => '...', 'routing_key' => '...', + // 'queue_name' => '...', 'redelivered' => false, 'delivery_tag' => 1] + // // GCP event: // ['kind' => 'gcp', 'trigger_path' => '...', 'payload' => '...', 'message_id' => '...', 'subscription' => '...', // 'ordering_key' => '...', 'attributes' => [...], 'delivery_type' => 'push', @@ -1435,6 +1461,30 @@ export const INITIAL_CODE = { // for related places search: ADD_NEW_LANG } +/** + * Whether a bash script body runs inside a custom container image that does not + * ship the `wmill` CLI (nor `jq`), namely `# sandbox ` or `# docker`. In + * that case editor snippets must fall back to a plain HTTP client instead of `wmill`. + * + * Mirrors the worker's annotation grammar (backend/windmill-common/src/worker.rs, + * `BashAnnotations`): only leading comment lines are scanned, stopping at the first + * non-comment line. A bare `# sandbox` (no image) is the nsjail-bash modifier that + * still runs on the worker rootfs where `wmill` is available, so it is excluded. + */ +export function bashRunsInCustomImage(code: string): boolean { + for (const line of code.split('\n')) { + const trimmed = line.trim() + if (trimmed === '') continue + if (!trimmed.startsWith('#')) break + const tokens = trimmed.slice(1).trim().split(/\s+/) + // `# sandbox ` selects a container; bare `# sandbox` does not. + if (tokens[0] === 'sandbox' && tokens[1]) return true + // `# docker` (v1 daemon runtime) runs in the referenced image, no wmill. + if (tokens[0] === 'docker' && tokens.length === 1) return true + } + return false +} + export function isInitialCode(content: string): boolean { for (const lang of Object.values(INITIAL_CODE)) { for (const code of Object.values(lang)) { diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 0449796d21..72526a3819 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -44,6 +44,7 @@ export interface UserWorkspace { parent_workspace_id?: string | null is_dev_workspace?: boolean dev_workspace_label?: string | null + created_by?: string | null disabled: boolean } @@ -145,6 +146,15 @@ export const aiUserDisabled = writable( export const usedTriggerKinds = writable([]) export let globalDbManagerDrawer: StateStore = { val: undefined } +/** Read-only S3 file browser (S3FilePicker instance) mounted in the logged + * layout, used by Explore buttons in contexts that don't wire their own picker + * instance. Typed loosely because the component instance type resolves + * differently in .ts and .svelte contexts. */ +export let globalS3FilePickerExplorer: StateStore< + { open: (fileKey?: any, opts?: { s3ResourcePath?: string }) => Promise } | undefined +> = createState({ + val: undefined +}) export let globalForkModal: StateStore = createState({ val: undefined }) diff --git a/frontend/src/lib/userDraft.svelte.ts b/frontend/src/lib/userDraft.svelte.ts index 6ecacbce22..24572ee1ab 100644 --- a/frontend/src/lib/userDraft.svelte.ts +++ b/frontend/src/lib/userDraft.svelte.ts @@ -27,6 +27,7 @@ export const USER_DRAFT_ITEM_KINDS = [ 'trigger_kafka', 'trigger_nats', 'trigger_mqtt', + 'trigger_amqp', 'trigger_sqs', 'trigger_gcp', 'trigger_azure', diff --git a/frontend/src/lib/userDraftDbMigration.ts b/frontend/src/lib/userDraftDbMigration.ts index 4c3079ea94..1496e595cb 100644 --- a/frontend/src/lib/userDraftDbMigration.ts +++ b/frontend/src/lib/userDraftDbMigration.ts @@ -46,6 +46,7 @@ const ITEM_KINDS = [ 'trigger_kafka', 'trigger_nats', 'trigger_mqtt', + 'trigger_amqp', 'trigger_sqs', 'trigger_gcp', 'trigger_azure', diff --git a/frontend/src/lib/userDraftDbSyncer.svelte.ts b/frontend/src/lib/userDraftDbSyncer.svelte.ts index a75f6b6837..0e8257023d 100644 --- a/frontend/src/lib/userDraftDbSyncer.svelte.ts +++ b/frontend/src/lib/userDraftDbSyncer.svelte.ts @@ -1,7 +1,7 @@ import { SvelteMap } from 'svelte/reactivity' import { DraftService, type UserDraftItemKind } from './gen' import { OpenAPI } from './gen/core/OpenAPI' -import { createCoalescingKeyedRunner } from './coalescingRunner.svelte' +import { createCoalescingKeyedRunner, CoalescingDisplacedError } from './coalescingRunner.svelte' import { createDebouncerByKey } from './debouncerByKey.svelte' import { setLocalDraftHint } from './localDraftHints.svelte' @@ -92,10 +92,16 @@ export type UserDraftDbSyncerSaveOpts = { value: unknown | null /** Bypass the debouncer: cancel any pending autosave for this key (it * would otherwise overwrite what we send), route through the coalescing - * runner to preserve ordering against an in-flight POST, and resolve - * the returned promise only once the POST lands. Use for + * runner to preserve ordering against an in-flight POST, and resolve only + * once the key's save chain has drained. Use for * `await save(...); read-the-server` flows where a fire-and-forget save - * would race the next read. */ + * would race the next read. + * + * Resolving means "the key is settled", NOT "your payload won": a newer + * save can displace this one (it then carries the later state), and — as + * with every other `save` — `postSave` routes a rejected or failed POST to + * `conflicts` / `failures` rather than throwing. Read those to know what + * actually landed. */ immediate?: boolean /** Skip the optimistic-concurrency check and overwrite the server row. * Used by the conflict-resolution UI ("Overwrite the remote"). Default @@ -231,6 +237,25 @@ const flushes = new SvelteMap() */ const saveListeners = new Map void>>() +/** + * Global listeners fired whenever ANY draft write lands on the server — + * upserts and deletes alike. This is the invalidation hook for caches keyed + * on persisted draft state (the chat diff snapshot): the moment a save + * commits, the affected item can be marked stale without polling. + */ +type DraftSavedEvent = { workspace: string; itemKind: UserDraftItemKind; path: string } +const anySavedListeners = new Set<(event: DraftSavedEvent) => void>() + +function notifyAnySaved(event: DraftSavedEvent): void { + for (const listener of [...anySavedListeners]) { + try { + listener(event) + } catch (e) { + console.error('UserDraftDbSyncer.onAnySaved listener threw', e) + } + } +} + /** * Best-effort error → readable string. The generated client wraps HTTP * failures as `ApiError` (`body` / `statusText`); raw fetch errors are a @@ -304,6 +329,10 @@ async function postSave(opts: UserDraftDbSyncerSaveOpts): Promise { const listeners = saveListeners.get(key) if (listeners) for (const l of [...listeners]) l() } + // Global subscribers hear deletes too — a removed row invalidates + // cached state the same way an upsert does. Listener errors must never + // make a committed save read as failed. + notifyAnySaved({ workspace: opts.workspace, itemKind: opts.itemKind, path: opts.path }) } catch (e) { console.error('UserDraftDbSyncer.save failed', e) // Leave pending opts in place so the next attempt retries the same @@ -360,8 +389,11 @@ function flushOnPageHide(): void { console.error('UserDraftDbSyncer: keepalive flush failed', e) }) // POST advanced the row past `lastSync` and we can't read the - // response — mark the key so a bfcache restore drops it. + // response — mark the key so a bfcache restore drops it, and notify + // subscribers conservatively (this path bypasses postSave; on a + // bfcache restore a cache must not serve the pre-flush state). staleSyncAfterHideFlush.add(key) + notifyAnySaved({ workspace: opts.workspace, itemKind: opts.itemKind, path: opts.path }) } catch (e) { console.error('UserDraftDbSyncer: keepalive flush threw', e) } @@ -441,10 +473,19 @@ export const UserDraftDbSyncer = { pendingSaveOpts.set(key, opts) if (opts.immediate) { // Drop the queued autosave — firing it after our POST would - // re-save the pre-delete value. + // re-save the pre-delete value. `submitAndWait` displaces the + // runner's own pending task, so no `runner.cancel` needed. debouncer.cancel(key) - runner.cancel(key) - await runner.submitAndWait(key, () => postSave(opts)) + try { + await runner.submitAndWait(key, () => postSave(opts)) + } catch (e) { + // Displacement is not a failure: a newer save took our slot, so + // re-POSTing ours would undo it. Wait for the chain instead — + // callers await this to know the key is settled, not to know + // their own payload won. + if (!(e instanceof CoalescingDisplacedError)) throw e + await runner.settled(key) + } return } // Auto-save off: opts stay parked (above) for an explicit flush but @@ -540,6 +581,18 @@ export const UserDraftDbSyncer = { } }, + /** + * Fires when any draft write lands on the server — upserts AND deletes, + * every workspace and key. For caches over persisted draft state that + * must invalidate the affected item the moment a write commits. + */ + onAnySaved(listener: (event: DraftSavedEvent) => void): () => void { + anySavedListeners.add(listener) + return () => { + anySavedListeners.delete(listener) + } + }, + /** Reactive conflict snapshot (if any) for a draft. */ getConflict(query: UserDraftLastSyncQuery): { readonly conflict: DraftConflictInfo | undefined @@ -563,8 +616,10 @@ export const UserDraftDbSyncer = { /** * Force-save: bypass the `last_sync` check and overwrite the server row - * (conflict modal's "Overwrite the remote"). Resolves only after the - * POST lands so the caller can `await` before navigating / refetching. + * (conflict modal's "Overwrite the remote"). Resolves once the key's save + * chain drains — see `immediate`; resolution means the chain settled, not + * that this force payload won (a later save can displace it). Callers + * `await` before navigating / refetching. */ async overwrite(opts: Omit): Promise { await this.save({ ...opts, immediate: true, force: true }) @@ -572,8 +627,10 @@ export const UserDraftDbSyncer = { /** * Flush the draft's queued autosave NOW (explicit Ctrl/Cmd+S). Re-submits - * the parked opts with `immediate: true` and resolves only after the POST - * lands, so callers can `await flush(...); show "Saved"`. + * the parked opts with `immediate: true` and resolves once the key's save + * chain drains (see `immediate` — the parked payload may be displaced by a + * later save carrying newer state), so callers can `await flush(...); show + * "Saved"`. * * No-op when nothing is pending. "No pending" does NOT mean "nothing to * save" — Monaco may hold unmaterialized text; flush the editor diff --git a/frontend/src/lib/userDraftDisplacedSave.test.ts b/frontend/src/lib/userDraftDisplacedSave.test.ts new file mode 100644 index 0000000000..50e24ba5c6 --- /dev/null +++ b/frontend/src/lib/userDraftDisplacedSave.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect, afterEach, vi } from 'vitest' + +// Mocked so a test can hold a POST in flight — that window is what makes a +// queued save displaceable. +const updateDraft = vi.fn(async (..._args: any[]) => ({ + status: 'saved' as const, + current_timestamp: '2020-01-01T00:00:00Z' +})) + +vi.mock('./gen', () => ({ + DraftService: { updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) } +})) +vi.mock('./gen/core/OpenAPI', () => ({ OpenAPI: { BASE: '' } })) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) + +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' + +function deferred() { + let resolve!: (v: T) => void + const promise = new Promise((res) => (resolve = res)) + return { promise, resolve } +} + +afterEach(() => { + vi.clearAllMocks() + updateDraft.mockResolvedValue({ status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' }) +}) + +/** + * Deploying queues several saves for one draft key back-to-back (mirror write, + * post-deploy delete, unmount flush), so the runner displaces one of them. A + * displaced save must read as "superseded", never as a failure. + */ +describe('UserDraftDbSyncer immediate save displacement', () => { + it('resolves a displaced immediate save once the superseding save lands', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_a' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + // Queues behind `first`, then gets displaced by the delete below. + const displaced = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + + inFlight.resolve() + await expect(displaced).resolves.toBeUndefined() + await Promise.all([first, deleting]) + + // The displaced task never POSTed — the delete carries the later state. + expect(updateDraft).toHaveBeenCalledTimes(2) + expect(updateDraft.mock.calls.map((c: any[]) => c[0].requestBody.value)).toEqual([ + { content: '1' }, + null + ]) + }) + + it('does not resolve a displaced save before the superseding POST lands', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_b' } + const inFlight = deferred() + const deletePost = deferred() + updateDraft + .mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + .mockImplementationOnce(async () => { + await deletePost.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:01Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + const displaced = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + + let displacedSettled = false + void displaced.then(() => (displacedSettled = true)) + + inFlight.resolve() + await vi.waitFor(() => expect(updateDraft).toHaveBeenCalledTimes(2)) + // Delete still in flight: callers that `await save()` before invalidating + // must not read the server yet. + expect(displacedSettled).toBe(false) + + deletePost.resolve() + await Promise.all([first, displaced, deleting]) + expect(displacedSettled).toBe(true) + }) + + it('resolves a pending save dropped by lockSync without POSTing it', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_lock' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + const first = UserDraftDbSyncer.save({ ...q, value: { content: '1' }, immediate: true }) + const dropped = UserDraftDbSyncer.save({ ...q, value: { content: '2' }, immediate: true }) + // Another user's draft was loaded: this value must never reach the server. + UserDraftDbSyncer.lockSync(q) + + inFlight.resolve() + // Resolves like every other save on a locked key — the lock's whole point + // is that the write is dropped, so the caller has nothing to wait for. + await expect(dropped).resolves.toBeUndefined() + await first + expect(updateDraft).toHaveBeenCalledTimes(1) + expect(updateDraft.mock.calls[0][0].requestBody.value).toEqual({ content: '1' }) + UserDraftDbSyncer.unlockSync(q) + }) + + it('resolves a flush displaced by a later immediate save', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/displaced_c' } + const inFlight = deferred() + updateDraft.mockImplementationOnce(async () => { + await inFlight.promise + return { status: 'saved', current_timestamp: '2020-01-01T00:00:00Z' } + }) + + // Park opts (the reactive mirror's autosave) so `flush` has something to send. + void UserDraftDbSyncer.save({ ...q, value: { content: 'typed' }, auto: true }) + const first = UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, immediate: true }) + const flushed = UserDraftDbSyncer.flush(q) // pending behind `first` + const deleting = UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) // displaces it + + inFlight.resolve() + await expect(flushed).resolves.toBeUndefined() + await Promise.all([first, deleting]) + }) +}) diff --git a/frontend/src/lib/userDraftFlushToggle.test.ts b/frontend/src/lib/userDraftFlushToggle.test.ts new file mode 100644 index 0000000000..6d3ab29f77 --- /dev/null +++ b/frontend/src/lib/userDraftFlushToggle.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, afterEach, vi } from 'vitest' + +const updateDraft = vi.fn(async (..._args: any[]) => ({ + status: 'saved' as const, + current_timestamp: '2020-01-01T00:00:00Z' +})) + +vi.mock('./gen', () => ({ + DraftService: { updateDraft: (...a: unknown[]) => updateDraft(...(a as [])) } +})) +vi.mock('./gen/core/OpenAPI', () => ({ OpenAPI: { BASE: '' } })) +vi.mock('./localDraftHints.svelte', () => ({ setLocalDraftHint: vi.fn() })) + +import { UserDraftDbSyncer } from './userDraftDbSyncer.svelte' + +afterEach(() => { + vi.clearAllMocks() + UserDraftDbSyncer.autosaveEnabled = true +}) + +/** + * Read-only consumers (the chat `diff` tool) flush with `honorAutosaveToggle` + * and then read `hasUnsavedDisabledChanges` to know the persisted state is + * stale. Pins the contract pair: a toggle-honoring flush must NOT persist + * auto-save-off edits (and must keep reporting them), while an explicit flush + * persists them and clears the signal. + */ +describe('UserDraftDbSyncer toggle-honoring flush', () => { + it('keeps auto-save-off edits parked and reported; explicit flush clears them', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/toggle_off' } + UserDraftDbSyncer.autosaveEnabled = false + await UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, auto: true, canBeDisabled: true }) + + await UserDraftDbSyncer.flush(q, { honorAutosaveToggle: true }) + expect(updateDraft).not.toHaveBeenCalled() + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(true) + + await UserDraftDbSyncer.flush(q) + expect(updateDraft).toHaveBeenCalledTimes(1) + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(false) + }) +}) + +/** A conflicted save leaves the local payload parked with the conflict + * recorded while the state reads 'none' — the exact triple the diff tool's + * unflushed-edits detection reads. Pins that a conflict never looks like a + * clean sync. */ +describe('UserDraftDbSyncer conflict aftermath', () => { + it('keeps the payload parked and the conflict readable after flush settles', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/conflicted' } + updateDraft.mockResolvedValueOnce({ + status: 'conflict', + current_timestamp: '2020-01-02T00:00:00Z' + }) + await UserDraftDbSyncer.save({ ...q, value: { content: 'mine' }, immediate: true }) + + // The triple the diff tool's unflushed detection reads: conflict set, + // state 'none' (not pending/failed), auto-save signal silent. + expect(UserDraftDbSyncer.getConflict(q).conflict).toBeDefined() + expect(UserDraftDbSyncer.getState(q).state).toBe('none') + expect(UserDraftDbSyncer.hasUnsavedDisabledChanges(q)).toBe(false) + }) +}) + +/** The diff snapshot cache invalidates through this hook — it must fire for + * upserts AND deletes, the moment the write lands. */ +describe('UserDraftDbSyncer.onAnySaved', () => { + it('fires for landed upserts and deletes, and unsubscribes cleanly', async () => { + const q = { workspace: 'w', itemKind: 'script' as const, path: 'u/me/hooked' } + const events: string[] = [] + const off = UserDraftDbSyncer.onAnySaved((e) => events.push(`${e.itemKind}:${e.path}`)) + + await UserDraftDbSyncer.save({ ...q, value: { content: 'x' }, immediate: true }) + await UserDraftDbSyncer.save({ ...q, value: null, immediate: true }) + expect(events).toEqual(['script:u/me/hooked', 'script:u/me/hooked']) + + off() + await UserDraftDbSyncer.save({ ...q, value: { content: 'y' }, immediate: true }) + expect(events).toHaveLength(2) + }) +}) diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index fd57862f3b..3dd3c722bf 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -844,6 +844,21 @@ export function isMac(): boolean { return navigator.userAgent.indexOf('Mac OS X') !== -1 } +/** + * True on Chromium-based browsers (Chrome, Edge, Brave, Opera, ...). Gates + * capabilities that are only faithful on Blink, e.g. DOM screenshot capture. + * userAgentData is itself Chromium-only; the UA fallback covers Chromium + * versions predating it ("Chrome/" never appears in Gecko or WebKit UAs). + */ +export function isChromiumBrowser(): boolean { + if (typeof navigator === 'undefined') return false + const brands = (navigator as any).userAgentData?.brands + if (Array.isArray(brands)) { + return brands.some((entry) => typeof entry?.brand === 'string' && /chromium/i.test(entry.brand)) + } + return /chrome\//i.test(navigator.userAgent) +} + export function getModifierKey(): string { return isMac() ? '⌘' : 'Ctrl+' } diff --git a/frontend/src/lib/utils/featureUsage.test.ts b/frontend/src/lib/utils/featureUsage.test.ts new file mode 100644 index 0000000000..3e13ff17d7 --- /dev/null +++ b/frontend/src/lib/utils/featureUsage.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/gen', () => ({ OpenAPI: { BASE: '/api' } })) +vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: () => () => {} } })) + +import { createFeatureUsageBuffer, type FeatureUsageEventPayload } from './featureUsage' + +describe('createFeatureUsageBuffer', () => { + it('sums repeated events per (feature, kind, key, entity) and flushes one batch', async () => { + const send = vi.fn().mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' }) + buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' }) + buffer.log('ai_session', 'tokens', { entityId: 's1', value: 120 }) + buffer.log('ai_session', 'message', { key: 'global', entityId: 's2' }) + await buffer.flush() + + expect(send).toHaveBeenCalledTimes(1) + const [workspace, events] = send.mock.calls[0] + expect(workspace).toBe('ws1') + expect(events).toEqual( + expect.arrayContaining([ + { feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's1', value: 2 }, + { feature: 'ai_session', kind: 'tokens', key: '', entity_id: 's1', value: 120 }, + { feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's2', value: 1 } + ]) + ) + expect(events).toHaveLength(3) + + // Flushed events must not be re-sent. + await buffer.flush() + expect(send).toHaveBeenCalledTimes(1) + }) + + it('splits batches per workspace and drops events without any workspace', async () => { + const send = vi.fn().mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => undefined) + + buffer.log('ai_session', 'created', { key: 'fork' }) // no workspace -> dropped + buffer.log('ai_session', 'created', { key: 'fork', workspace: 'ws1' }) + buffer.log('ai_session', 'created', { key: 'root', workspace: 'ws2' }) + await buffer.flush() + + expect(send).toHaveBeenCalledTimes(2) + const workspaces = send.mock.calls.map((c) => c[0]).sort() + expect(workspaces).toEqual(['ws1', 'ws2']) + }) + + it('starts every chunk request before any send resolves (pagehide flush)', async () => { + const send = vi.fn().mockReturnValue(new Promise(() => {})) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + for (let i = 0; i < 60; i++) { + buffer.log('ai_session', 'tool', { key: `tool_${i}` }) + } + buffer.log('ai_session', 'message', { workspace: 'ws2' }) + void buffer.flush() + await Promise.resolve() + + // keepalive only protects requests that were issued; a sequential flush + // would have started just the first chunk here. + expect(send).toHaveBeenCalledTimes(3) + }) + + it('chunks flushes above the per-request cap and survives send failures', async () => { + const send = vi.fn().mockRejectedValueOnce(new Error('network')).mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + for (let i = 0; i < 60; i++) { + buffer.log('ai_session', 'tool', { key: `tool_${i}` }) + } + await expect(buffer.flush()).resolves.toBeUndefined() + + expect(send).toHaveBeenCalledTimes(2) + const sent = send.mock.calls.flatMap((c) => c[1] as FeatureUsageEventPayload[]) + expect(send.mock.calls[0][1]).toHaveLength(50) + expect(sent).toHaveLength(60) + }) +}) diff --git a/frontend/src/lib/utils/featureUsage.ts b/frontend/src/lib/utils/featureUsage.ts new file mode 100644 index 0000000000..6e02e4ce0e --- /dev/null +++ b/frontend/src/lib/utils/featureUsage.ts @@ -0,0 +1,137 @@ +import { get } from 'svelte/store' +import { OpenAPI } from '$lib/gen' +import { workspaceStore } from '$lib/stores' + +// Anonymous product-usage counters (e.g. AI session activity), batched into the +// backend `feature_usage` accumulator. Only aggregated counts ever leave the +// instance, and only when telemetry is enabled and not in minimal mode — never +// log paths, prompts, code, or user identifiers here (entity ids must be +// opaque random ids). + +export interface FeatureUsageOpts { + key?: string + entityId?: string + value?: number + /** Workspace whose API route carries the batch; defaults to the active workspace. */ + workspace?: string +} + +type SendFn = (workspace: string, events: FeatureUsageEventPayload[]) => Promise + +export interface FeatureUsageEventPayload { + feature: string + kind: string + key?: string + entity_id?: string + value?: number +} + +const FLUSH_INTERVAL_MS = 30_000 +// Backend caps a batch at 50 events; chunk larger flushes. +const MAX_EVENTS_PER_REQUEST = 50 + +export function createFeatureUsageBuffer( + send: SendFn, + getDefaultWorkspace: () => string | undefined, + flushIntervalMs = FLUSH_INTERVAL_MS +) { + // One accumulator per (workspace, feature, kind, key, entityId): repeated + // events sum locally so a chatty UI still produces one upsert per flush. + const pending = new Map() + let timer: ReturnType | undefined + + function log(feature: string, kind: string, opts: FeatureUsageOpts = {}): void { + const workspace = opts.workspace ?? getDefaultWorkspace() + if (!workspace) return + const key = opts.key ?? '' + const entityId = opts.entityId ?? '' + const value = Math.max(1, Math.round(opts.value ?? 1)) + const mapKey = `${workspace}\u0000${feature}\u0000${kind}\u0000${key}\u0000${entityId}` + const existing = pending.get(mapKey) + if (existing) { + existing.event.value = (existing.event.value ?? 1) + value + } else { + pending.set(mapKey, { + workspace, + event: { feature, kind, key, entity_id: entityId, value } + }) + } + if (timer === undefined) { + timer = setTimeout(() => { + timer = undefined + void flush() + }, flushIntervalMs) + } + } + + async function flush(): Promise { + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + if (pending.size === 0) return + const byWorkspace = new Map() + for (const { workspace, event } of pending.values()) { + let events = byWorkspace.get(workspace) + if (!events) { + events = [] + byWorkspace.set(workspace, events) + } + events.push(event) + } + pending.clear() + // Start every chunk request synchronously before awaiting: the pagehide + // flush only protects requests that were already issued (keepalive can't + // help a fetch that never started). + const inflight: Promise[] = [] + for (const [workspace, events] of byWorkspace) { + for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) { + inflight.push( + send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)).catch(() => { + // Telemetry is best-effort: drop the batch rather than retry. + }) + ) + } + } + await Promise.all(inflight) + } + + return { log, flush } +} + +const buffer = createFeatureUsageBuffer( + async (workspace, events) => { + // Raw fetch instead of the generated client: `keepalive` lets the request + // finish after tab close/navigation, which is when the final flush runs. + // Auth rides on the token cookie (WITH_CREDENTIALS app setup). + await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/workspaces/log_feature_usage`, { + method: 'POST', + credentials: 'include', + keepalive: true, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ events }) + }) + }, + () => get(workspaceStore) ?? undefined +) + +if (typeof document !== 'undefined') { + // Flush what's buffered before the tab goes away. pagehide covers + // close/navigation paths where visibilitychange is not delivered. + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden') { + void buffer.flush() + } + }) + window.addEventListener('pagehide', () => { + void buffer.flush() + }) +} + +/** + * Record an anonymous feature-usage event. Fire-and-forget: events are summed + * locally per (feature, kind, key, entityId) and flushed in batches. + */ +export function logFeatureUsage(feature: string, kind: string, opts: FeatureUsageOpts = {}): void { + buffer.log(feature, kind, opts) +} diff --git a/frontend/src/lib/utils/workspaceHierarchy.ts b/frontend/src/lib/utils/workspaceHierarchy.ts index e1ff76f934..9c817f49e2 100644 --- a/frontend/src/lib/utils/workspaceHierarchy.ts +++ b/frontend/src/lib/utils/workspaceHierarchy.ts @@ -138,6 +138,21 @@ export function workspaceIsFork( return allWorkspaces.find((w) => w.id === workspaceId)?.parent_workspace_id != null } +/** + * Whether `userEmail` is the creator of a fork workspace. The fork creator gets workspace-settings + * access (the fork members screen) even when they are not an admin of it: forking as an ordinary + * developer copies their parent `usr` row, leaving them otherwise unable to bring collaborators in. + * Mirrors the backend `authorize_fork_owner_add_user` grant. + */ +export function isForkOwner( + workspace: UserWorkspace | undefined, + userEmail: string | null | undefined +): boolean { + return ( + Boolean(workspace?.parent_workspace_id) && !!userEmail && workspace?.created_by === userEmail + ) +} + /** * The canonical dev workspace of a prod workspace, if any (at most one per prod). Used to redirect * edits from a locked prod workspace into its dev workspace. Disabled dev workspaces are excluded: diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 556ac0a3c7..7d517c3bb4 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -6,6 +6,7 @@ import { HttpTriggerService, KafkaTriggerService, MqttTriggerService, + AmqpTriggerService, NatsTriggerService, PostgresTriggerService, ScheduleService, @@ -36,6 +37,7 @@ export type Kind = | 'nats_trigger' | 'postgres_trigger' | 'mqtt_trigger' + | 'amqp_trigger' | 'sqs_trigger' | 'gcp_trigger' | 'azure_trigger' @@ -94,6 +96,8 @@ export async function existsTrigger( return await KafkaTriggerService.existsKafkaTrigger(data) } else if (triggerKind === 'mqtt') { return await MqttTriggerService.existsMqttTrigger(data) + } else if (triggerKind === 'amqp') { + return await AmqpTriggerService.existsAmqpTrigger(data) } else if (triggerKind === 'postgres') { return await PostgresTriggerService.existsPostgresTrigger(data) } else if (triggerKind === 'sqs') { @@ -196,6 +200,21 @@ export async function getTriggersDeployData( createFn: MqttTriggerService.createMqttTrigger, updateFn: MqttTriggerService.updateMqttTrigger } + } else if (kind === 'amqp') { + const amqpTrigger = await AmqpTriggerService.getAmqpTrigger({ + workspace: workspace!, + path: path + }) + + return { + data: { + ...amqpTrigger, + permissioned_as: onBehalfOf, + preserve_permissioned_as: preservePermissionedAs + }, + createFn: AmqpTriggerService.createAmqpTrigger, + updateFn: AmqpTriggerService.updateAmqpTrigger + } } else if (kind === 'nats') { const natsTrigger = await NatsTriggerService.getNatsTrigger({ workspace: workspace!, @@ -372,6 +391,8 @@ export async function getTriggerValue(kind: TriggerKind, path: string, workspace trigger = await KafkaTriggerService.getKafkaTrigger({ workspace, path }) } else if (kind === 'mqtt') { trigger = await MqttTriggerService.getMqttTrigger({ workspace, path }) + } else if (kind === 'amqp') { + trigger = await AmqpTriggerService.getAmqpTrigger({ workspace, path }) } else if (kind === 'nats') { trigger = await NatsTriggerService.getNatsTrigger({ workspace, path }) } else if (kind === 'postgres') { @@ -412,6 +433,9 @@ export async function getTriggerPermissionedAs( } else if (kind === 'mqtt') { const trigger = await MqttTriggerService.getMqttTrigger({ workspace, path }) return trigger.permissioned_as + } else if (kind === 'amqp') { + const trigger = await AmqpTriggerService.getAmqpTrigger({ workspace, path }) + return trigger.permissioned_as } else if (kind === 'nats') { const trigger = await NatsTriggerService.getNatsTrigger({ workspace, path }) return trigger.permissioned_as @@ -493,6 +517,13 @@ export async function getTriggerDependency(kind: TriggerKind, path: string, work }) result = retrieveKindsValues({ resource_path: mqtt_resource_path, script_path, is_flow }) + } else if (kind === 'amqp') { + const { amqp_resource_path, script_path, is_flow } = await AmqpTriggerService.getAmqpTrigger({ + workspace: workspace!, + path: path + }) + + result = retrieveKindsValues({ resource_path: amqp_resource_path, script_path, is_flow }) } else if (kind === 'nats') { const { nats_resource_path, script_path, is_flow } = await NatsTriggerService.getNatsTrigger({ workspace: workspace!, diff --git a/frontend/src/lib/utils_draft_deploy.ts b/frontend/src/lib/utils_draft_deploy.ts index ee8538e4db..514ce9cdb2 100644 --- a/frontend/src/lib/utils_draft_deploy.ts +++ b/frontend/src/lib/utils_draft_deploy.ts @@ -27,6 +27,7 @@ import { KafkaTriggerService, NatsTriggerService, MqttTriggerService, + AmqpTriggerService, SqsTriggerService, GcpTriggerService, AzureTriggerService, @@ -38,7 +39,9 @@ import type { DeployResult } from '$lib/utils_workspace_deploy' import { TRIGGER_RUNTIME_IGNORE } from '$lib/utils_deployable' import { deployRawAppDraft } from '$lib/rawAppDeploy' import { canonicalRawAppDiffValue } from '$lib/components/raw_apps/utils' +import { classicAppDraftParts } from '$lib/appDiffSides' import { invalidateWorkspaceDrafts } from '$lib/workspaceDrafts.svelte' +import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' import { setLocalDraftHint } from '$lib/localDraftHints.svelte' import { userStore } from '$lib/stores' import { deployTriggers, type Trigger } from '$lib/components/triggers/utils' @@ -49,6 +52,7 @@ import { savePostgresTriggerFromCfg } from '$lib/components/triggers/postgres/ut import { saveKafkaTriggerFromCfg } from '$lib/components/triggers/kafka/utils' import { saveNatsTriggerFromCfg } from '$lib/components/triggers/nats/utils' import { saveMqttTriggerFromCfg } from '$lib/components/triggers/mqtt/utils' +import { saveAmqpTriggerFromCfg } from '$lib/components/triggers/amqp/utils' import { saveSqsTriggerFromCfg } from '$lib/components/triggers/sqs/utils' import { saveGcpTriggerFromCfg } from '$lib/components/triggers/gcp/utils' import { saveAzureTriggerFromCfg } from '$lib/components/triggers/azure/utils' @@ -79,6 +83,8 @@ const OVERLAY_GETTERS: Partial< NatsTriggerService.getNatsTrigger({ workspace, path, getDraft: true }), trigger_mqtt: (workspace, path) => MqttTriggerService.getMqttTrigger({ workspace, path, getDraft: true }), + trigger_amqp: (workspace, path) => + AmqpTriggerService.getAmqpTrigger({ workspace, path, getDraft: true }), trigger_sqs: (workspace, path) => SqsTriggerService.getSqsTrigger({ workspace, path, getDraft: true }), trigger_gcp: (workspace, path) => @@ -90,21 +96,35 @@ const OVERLAY_GETTERS: Partial< } /** Strip the per-user draft-overlay metadata, returning `{deployed, draft}`. */ -function splitOverlay(r: any): { deployed: any; draft: any } { +function splitOverlay(r: any): { + deployed: any + draft: any + hasDraft: boolean + noDeployed: boolean +} { const { draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, ...deployed } = r - return { deployed, draft: draft ?? deployed } + return { + deployed, + draft: draft ?? deployed, + hasDraft: draft != null, + noDeployed: no_deployed === true + } } export interface DraftDiffValues { deployed: unknown draft: unknown + /** False when the overlay carried no draft row (the item's own value was used as the draft side). */ + hasDraft: boolean + /** True when the item has never been deployed (`draft_only` overlay). */ + noDeployed: boolean } // Empty-but-valid "deployed" shapes for draft_only items. A bare `{}` breaks @@ -113,7 +133,56 @@ export interface DraftDiffValues { const EMPTY_DEPLOYED: Partial unknown>> = { script: (draft) => ({ content: '', language: draft?.language, schema: {} }), flow: () => ({ summary: '', value: { modules: [] }, schema: {} }), - app: () => ({ summary: '', value: {}, policy: {} }) + app: () => ({ summary: '', value: {} }) +} + +// Server-managed script-row fields, stripped from BOTH sides of a draft diff: +// never user-edited, they are either identical noise (created_at, workspace_id) +// or spuriously different (lock is recomputed at deploy). The draft-side +// pinned-base `parent_hash` is stripped separately, like the flow `version_id`. +const SCRIPT_ROW_RUNTIME_IGNORE = new Set([ + 'workspace_id', + 'hash', + 'parent_hash', + 'parent_hashes', + 'created_at', + 'created_by', + 'archived', + 'deleted', + 'extra_perms', + 'lock', + 'lock_error_logs', + 'starred', + 'has_draft', + 'draft_only', + 'assets', + 'marked' +]) + +function stripScriptRowRuntime(row: any): Record { + if (!row || typeof row !== 'object') return {} + return Object.fromEntries(Object.entries(row).filter(([k]) => !SCRIPT_ROW_RUNTIME_IGNORE.has(k))) +} + +/** Canonicalize a raw draft value onto the same shape `getDraftDiffValues` + * yields for its draft side, so a value read from an in-memory editor cell + * diffs cleanly against a deployed side (and compares equal to its own + * persisted form instead of differing on stripped fields). */ +export function canonicalDraftSideValue(kind: DraftKind, value: unknown): unknown { + if (kind === 'script') return stripScriptRowRuntime(value) + if (kind === 'raw_app') return canonicalRawAppDiffValue((value ?? {}) as Record) + if (kind === 'app') { + const parts = classicAppDraftParts(value) + return { summary: parts.summary ?? '', value: parts.value } + } + if (kind === 'flow' && value !== null && typeof value === 'object') { + const { version_id: _v, ...rest } = value as Record + return rest + } + // Drawer kinds (variables/resources/schedules/triggers): the editor-state + // shape diverges from the backend row — same canonicalization the overlay + // diff applies. + return canonicalizeDraftDiffValue(kind, value, true) } // Schedule & trigger rows drop the same runtime/server-managed fields as the @@ -189,15 +258,17 @@ export async function getDraftDiffValues( draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, hash: _h, ...deployed } = r - const draftValue = draft ?? deployed + const draftValue = stripScriptRowRuntime(draft ?? deployed) return { - deployed: draftOnly ? EMPTY_DEPLOYED.script!(draftValue) : deployed, - draft: draftValue + deployed: draftOnly ? EMPTY_DEPLOYED.script!(draftValue) : stripScriptRowRuntime(deployed), + draft: draftValue, + hasDraft: draft != null, + noDeployed: no_deployed === true } } else if (kind === 'flow') { const r = (await FlowService.getFlowByPath({ workspace, path, getDraft: true })) as any @@ -205,7 +276,7 @@ export async function getDraftDiffValues( draft, is_draft: _i, draft_saved_at: _c, - no_deployed: _n, + no_deployed, other_drafts_users: _o, version_id: _v, ...deployed @@ -213,7 +284,12 @@ export async function getDraftDiffValues( // Strip the draft's pinned base `version_id` (which differs from the deployed // head for a stale draft) so it never renders as a spurious diff line. const { version_id: _dv, ...draftValue } = (draft ?? deployed) as any - return { deployed: draftOnly ? EMPTY_DEPLOYED.flow!(draftValue) : deployed, draft: draftValue } + return { + deployed: draftOnly ? EMPTY_DEPLOYED.flow!(draftValue) : deployed, + draft: draftValue, + hasDraft: draft != null, + noDeployed: no_deployed === true + } } else if (kind === 'app' || kind === 'raw_app') { // A never-deployed raw app has no `app` row; the backend resolves the // draft kind from `rawApp`, so it MUST be set or the lookup 404s. @@ -228,22 +304,36 @@ export async function getDraftDiffValues( // deployed row nests them under `value`, and deployed inline scripts carry // server-recomputed locks. Canonicalize both onto the same shape with the // post-deploy noise stripped — the same module the editor's Diff button uses. + // A staged rename (`draft_path`) changes where deploy lands the app — + // compare it as `path` on both sides so a rename-only draft diffs. + const rawDraftPath = (r.draft?.draft_path as string | undefined) ?? r.path return { - deployed: draftOnly ? canonicalRawAppDiffValue({}) : canonicalRawAppDiffValue(r), - draft: canonicalRawAppDiffValue(r.draft ?? r) + deployed: draftOnly + ? canonicalRawAppDiffValue({}) + : { ...canonicalRawAppDiffValue(r), path: r.path }, + draft: { ...canonicalRawAppDiffValue(r.draft ?? r), path: rawDraftPath }, + hasDraft: r.draft != null, + noDeployed: r.no_deployed === true } } - const deployed = { - summary: r.summary, - value: r.value, - policy: r.policy, - path: r.path, - custom_path: r.custom_path + // Classic app: the editor drafts the bare grid with summary/draft_path + // mirrored into it, while the row keeps summary as a column beside + // `value`. Both sides reduce to `{ summary, value }` with the metadata + // extracted from the grid, so a summary edit diffs as a summary edit and + // the grid never diffs against draft-only markers. + const deployedParts = classicAppDraftParts(r.value) + const draftParts = r.draft != null ? classicAppDraftParts(r.draft) : deployedParts + const deployed = { summary: r.summary ?? '', value: deployedParts.value, path: r.path } + return { + deployed: draftOnly ? EMPTY_DEPLOYED.app!(undefined) : deployed, + draft: { + summary: draftParts.summary ?? r.summary ?? '', + value: draftParts.value, + path: draftParts.draftPath ?? r.path + }, + hasDraft: r.draft != null, + noDeployed: r.no_deployed === true } - // Strip the draft's pinned fork-base `parent_version` (the deployed allowlist - // above already omits it) so it never renders as a spurious diff line. - const { parent_version: _pv, ...draftValue } = (r.draft ?? deployed) as any - return { deployed: draftOnly ? EMPTY_DEPLOYED.app!(draftValue) : deployed, draft: draftValue } } else { // Variables / resources / schedules / triggers: one overlay GET yields // both sides, but the draft side is the editor's state shape while the @@ -254,10 +344,12 @@ export async function getDraftDiffValues( if (!getter) { throw new Error(`Draft diff not supported for kind ${kind}`) } - const { deployed, draft } = splitOverlay(await getter(workspace, path)) + const { deployed, draft, hasDraft, noDeployed } = splitOverlay(await getter(workspace, path)) return { deployed: draftOnly ? {} : canonicalizeDraftDiffValue(kind, deployed, false), - draft: canonicalizeDraftDiffValue(kind, draft, true) + draft: canonicalizeDraftDiffValue(kind, draft, true), + hasDraft, + noDeployed } } } @@ -552,6 +644,10 @@ export async function deployDraft( }) // Mutated the workspace's Server Drafts — refresh every mounted reader. invalidateWorkspaceDrafts(workspace) + // The DEPLOYED state moved: cached fork comparisons involving this + // workspace (as fork or as parent) are no longer trustworthy. Draft-only + // mutations skip this — they never move the deployed tally. + invalidateWorkspaceComparison(workspace) // For script/flow/app the server-side delete bypasses UserDraftDbSyncer, // so the syncer-owned hint won't auto-clear — clear it explicitly. // (Idempotent: the drawer-kind delete above already cleared it.) @@ -589,6 +685,8 @@ const TRIGGER_SAVERS: Partial< saveNatsTriggerFromCfg(p, cfg, edit, ws, writable([])), trigger_mqtt: (p, cfg, edit, ws) => saveMqttTriggerFromCfg(p, cfg, edit, ws, writable([])), + trigger_amqp: (p, cfg, edit, ws) => + saveAmqpTriggerFromCfg(p, cfg, edit, ws, writable([])), trigger_sqs: (p, cfg, edit, ws) => saveSqsTriggerFromCfg(p, cfg, edit, ws, writable([])), trigger_gcp: (p, cfg, edit, ws) => diff --git a/frontend/src/lib/workspaceComparison.test.ts b/frontend/src/lib/workspaceComparison.test.ts new file mode 100644 index 0000000000..bf9fc960b1 --- /dev/null +++ b/frontend/src/lib/workspaceComparison.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest' + +const compareWorkspaces = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { compareWorkspaces: (...a: unknown[]) => compareWorkspaces(...(a as [])) } +})) +vi.mock('$lib/stores', async () => { + const { writable } = await import('svelte/store') + return { usersWorkspaceStore: writable(undefined) } +}) + +import { fetchWorkspaceComparison, invalidateWorkspaceComparison } from './workspaceComparison' +import { usersWorkspaceStore } from '$lib/stores' + +function deferred() { + let resolve!: (v: T) => void + const promise = new Promise((res) => (resolve = res)) + return { promise, resolve } +} + +beforeEach(() => { + compareWorkspaces.mockReset() +}) + +describe('fetchWorkspaceComparison', () => { + it('a freshness-forced call never adopts an older in-flight request', async () => { + const first = deferred() + compareWorkspaces.mockImplementationOnce(() => first.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 1 } }) + + // Tolerant caller starts a request (e.g. the fork banner)... + const tolerant = fetchWorkspaceComparison('p', 'f-forced', { maxAgeMs: 30_000 }) + // ...a mutation happens, then a forced caller must get its OWN fetch. + const forced = fetchWorkspaceComparison('p', 'f-forced', { maxAgeMs: 0 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + first.resolve({ summary: { total_diffs: 0 } }) + expect((await forced).summary.total_diffs).toBe(1) + expect((await tolerant).summary.total_diffs).toBe(0) + }) + + it('a superseded older request never overwrites a newer result, even same-millisecond', async () => { + vi.useFakeTimers() + try { + const older = deferred() + compareWorkspaces.mockImplementationOnce(() => older.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 7 } }) + + // Same frozen Date.now() for both requests. + const tolerant = fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 30_000 }) + const forced = fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 0 }) + // Newer (forced) resolves FIRST; older resolves after with stale data. + expect((await forced).summary.total_diffs).toBe(7) + older.resolve({ summary: { total_diffs: 0 } }) + await tolerant + + // A tolerant read must see the newer result, not the late stale write. + const reread = await fetchWorkspaceComparison('p', 'f-race', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(7) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + + it('invalidation evicts by EITHER side and fences in-flight requests', async () => { + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 0 } }) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 3 } }) + + // Banner prewarms the cache pre-deploy... + await fetchWorkspaceComparison('p-side', 'f-inval', { maxAgeMs: 30_000 }) + // ...a deploy in the PARENT invalidates too... + invalidateWorkspaceComparison('p-side') + // ...so even a first-ever tolerant read cannot reuse the stale tally. + const fresh = await fetchWorkspaceComparison('p-side', 'f-inval', { maxAgeMs: 30_000 }) + expect(fresh.summary.total_diffs).toBe(3) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a pre-invalidation in-flight request is not joined and cannot land in the cache', async () => { + const stale = deferred() + compareWorkspaces.mockImplementationOnce(() => stale.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 9 } }) + + const preMutation = fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + invalidateWorkspaceComparison('f-fence') + // Tolerant post-mutation read: must NOT join the fenced request. + const post = fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + stale.resolve({ summary: { total_diffs: 0 } }) + await preMutation + expect((await post).summary.total_diffs).toBe(9) + // The stale request's late completion never landed in the cache. + const reread = await fetchWorkspaceComparison('p', 'f-fence', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(9) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a superseded request fenced by invalidation cannot repopulate the cache late', async () => { + const superseded = deferred() + compareWorkspaces.mockImplementationOnce(() => superseded.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 5 } }) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 6 } }) + + // Request A pends; forced request B replaces it in the inflight map and completes. + const a = fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 30_000 }) + await fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 0 }) + // Invalidation happens while A (no longer tracked in inflight) still pends. + invalidateWorkspaceComparison('f-late') + // A resolves late with pre-mutation data — it must NOT land in the cache. + superseded.resolve({ summary: { total_diffs: 0 } }) + await a + const read = await fetchWorkspaceComparison('p', 'f-late', { maxAgeMs: 30_000 }) + expect(read.summary.total_diffs).toBe(6) + }) + + it('tolerant callers join a recent in-flight request', async () => { + const first = deferred() + compareWorkspaces.mockImplementationOnce(() => first.promise) + + const a = fetchWorkspaceComparison('p', 'f-join', { maxAgeMs: 30_000 }) + const b = fetchWorkspaceComparison('p', 'f-join', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(1) + + first.resolve({ summary: { total_diffs: 2 } }) + expect((await a).summary.total_diffs).toBe(2) + expect((await b).summary.total_diffs).toBe(2) + }) + it('an account switch clears cached comparisons', async () => { + usersWorkspaceStore.set({ email: 'first@x.dev' } as any) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 3 } }) + await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + expect(compareWorkspaces).toHaveBeenCalledTimes(1) + + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 9 } }) + usersWorkspaceStore.set({ email: 'second@x.dev' } as any) + const fresh = await fetchWorkspaceComparison('p', 'f-owner', { maxAgeMs: 30_000 }) + expect(fresh.summary.total_diffs).toBe(9) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) + + it('a request started under the previous account never joins or lands after a switch', async () => { + const old = deferred() + compareWorkspaces.mockImplementationOnce(() => old.promise) + compareWorkspaces.mockResolvedValueOnce({ summary: { total_diffs: 5 } }) + + usersWorkspaceStore.set({ email: 'a@x.dev' } as any) + fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + usersWorkspaceStore.set({ email: 'b@x.dev' } as any) + const after = fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + // The new account must not have joined the old account's request. + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + + old.resolve({ summary: { total_diffs: 0 } }) + expect((await after).summary.total_diffs).toBe(5) + // And the old account's late result must not have landed in the cache. + const reread = await fetchWorkspaceComparison('p', 'f-switch', { maxAgeMs: 30_000 }) + expect(reread.summary.total_diffs).toBe(5) + expect(compareWorkspaces).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/src/lib/workspaceComparison.ts b/frontend/src/lib/workspaceComparison.ts new file mode 100644 index 0000000000..0d1ad8be25 --- /dev/null +++ b/frontend/src/lib/workspaceComparison.ts @@ -0,0 +1,181 @@ +/** + * Shared fetch layer for the fork↔parent workspace comparison + * (`compareWorkspaces`). The comparison is the expensive tally the fork banner, + * the compare page, and the chat `diff` tool all need — routing every consumer + * through this module means concurrent tolerant requests coalesce and a + * consumer that accepts a slightly stale result (`maxAgeMs`) can reuse the + * fetch another surface just made instead of recomputing it. + */ +import { get } from 'svelte/store' +import { WorkspaceService, type WorkspaceComparison } from '$lib/gen' +import { usersWorkspaceStore } from '$lib/stores' + +interface CacheEntry { + fetchedAt: number + generation: number + comparison: WorkspaceComparison +} + +interface InflightEntry { + startedAt: number + generation: number + promise: Promise +} + +const cache = new Map() +const inflight = new Map() +// Millisecond timestamps collide under concurrency — ordering between +// requests rides on this monotonic generation instead. +let requestGeneration = 0 +// Per-WORKSPACE floor: any request started at or below this generation is +// pre-invalidation and must neither be joined nor land in the cache. Keyed by +// workspace id (either side of a pair), so even requests the inflight map no +// longer tracks are fenced. +const invalidGenFloor = new Map() +// Raised when the authenticated identity changes — fences EVERY earlier +// request at once, including ones whose workspace ids nothing tracks anymore. +let globalGenFloor = 0 + +function generationFloor(parentWorkspaceId: string, forkWorkspaceId: string): number { + return Math.max( + globalGenFloor, + invalidGenFloor.get(parentWorkspaceId) ?? 0, + invalidGenFloor.get(forkWorkspaceId) ?? 0 + ) +} + +// Comparisons are permission-filtered per user but keyed only by workspace +// pair — an SPA logout/login must never serve one account's tallies (or let +// its in-flight fetches land) for another. +let cacheOwner: string | undefined = undefined + +function ensureCacheOwner(): void { + const owner = get(usersWorkspaceStore)?.email + if (owner === cacheOwner) return + cacheOwner = owner + cache.clear() + inflight.clear() + invalidGenFloor.clear() + globalGenFloor = requestGeneration +} +// Comparisons are big; keep only the few pairs a session actually browses. +const MAX_CACHE_ENTRIES = 8 + +function key(parentWorkspaceId: string, forkWorkspaceId: string): string { + return `${parentWorkspaceId}:${forkWorkspaceId}` +} + +/** + * Fetch (or reuse) the comparison of `forkWorkspaceId` against its parent. + * `maxAgeMs` (default 0) is the oldest result the caller accepts — applied to + * cached results AND to joining an in-flight request (a request is as old as + * its start). `maxAgeMs: 0` therefore always issues a fresh fetch: a caller + * forcing freshness after a mutation must never adopt a request that began + * before the mutation. + */ +export async function fetchWorkspaceComparison( + parentWorkspaceId: string, + forkWorkspaceId: string, + opts: { maxAgeMs?: number } = {} +): Promise { + return (await fetchWorkspaceComparisonMeta(parentWorkspaceId, forkWorkspaceId, opts)).comparison +} + +export interface WorkspaceComparisonMeta { + comparison: WorkspaceComparison + /** When the underlying request STARTED — a reused result is as old as its + * fetch, not as old as the reuse. Callers layering their own freshness + * window must age from this, or windows compound. */ + fetchedAt: number + /** Pass to `isComparisonCurrent` to learn whether an invalidation has + * outdated this result since. */ + generation: number +} + +/** True while no `invalidateWorkspaceComparison` (or identity change) has + * fenced the request that produced `generation`. */ +export function isComparisonCurrent( + parentWorkspaceId: string, + forkWorkspaceId: string, + generation: number +): boolean { + ensureCacheOwner() + return generation > generationFloor(parentWorkspaceId, forkWorkspaceId) +} + +export async function fetchWorkspaceComparisonMeta( + parentWorkspaceId: string, + forkWorkspaceId: string, + opts: { maxAgeMs?: number } = {} +): Promise { + ensureCacheOwner() + const k = key(parentWorkspaceId, forkWorkspaceId) + const maxAgeMs = opts.maxAgeMs ?? 0 + const cached = cache.get(k) + if (cached && Date.now() - cached.fetchedAt < maxAgeMs) { + return { + comparison: cached.comparison, + fetchedAt: cached.fetchedAt, + generation: cached.generation + } + } + const pending = inflight.get(k) + if ( + pending && + maxAgeMs > 0 && + Date.now() - pending.startedAt < maxAgeMs && + pending.generation > generationFloor(parentWorkspaceId, forkWorkspaceId) + ) { + return { + comparison: await pending.promise, + fetchedAt: pending.startedAt, + generation: pending.generation + } + } + const startedAt = Date.now() + const generation = ++requestGeneration + const run = (async () => { + const comparison = await WorkspaceService.compareWorkspaces({ + workspace: parentWorkspaceId, + targetWorkspaceId: forkWorkspaceId + }) + // A superseded (older-generation) or pre-invalidation request must not + // clobber a newer result. + const existing = cache.get(k) + if ( + generation > generationFloor(parentWorkspaceId, forkWorkspaceId) && + (!existing || existing.generation < generation) + ) { + cache.delete(k) + cache.set(k, { fetchedAt: startedAt, generation, comparison }) + // Insertion-ordered Map: evict the oldest pairs past the cap. + while (cache.size > MAX_CACHE_ENTRIES) { + cache.delete(cache.keys().next().value as string) + } + } + return comparison + })() + inflight.set(k, { startedAt, generation, promise: run }) + try { + return { comparison: await run, fetchedAt: startedAt, generation } + } finally { + if (inflight.get(k)?.promise === run) inflight.delete(k) + } +} + +/** Drop cached comparisons involving this workspace on EITHER side — a + * deploy in a parent moves its forks' tallies too. Also fences in-flight + * requests: nobody new joins them and their late results never land in the + * cache. (Workspace ids cannot contain ':', so the matches are exact.) */ +export function invalidateWorkspaceComparison(workspaceId: string): void { + const matches = (k: string) => k.startsWith(`${workspaceId}:`) || k.endsWith(`:${workspaceId}`) + for (const k of [...cache.keys()]) { + if (matches(k)) cache.delete(k) + } + for (const k of [...inflight.keys()]) { + if (matches(k)) inflight.delete(k) + } + // Fence EVERY request started before this point — including ones the + // inflight map no longer tracks (superseded requests still resolve late). + invalidGenFloor.set(workspaceId, requestGeneration) +} diff --git a/frontend/src/lib/workspaceDrafts.svelte.ts b/frontend/src/lib/workspaceDrafts.svelte.ts index 90a7cf9542..52e96ce7a1 100644 --- a/frontend/src/lib/workspaceDrafts.svelte.ts +++ b/frontend/src/lib/workspaceDrafts.svelte.ts @@ -47,13 +47,25 @@ export interface DraftItem { * `allUsers` listing surfaces other users' rows as `false` (view-only). * Defaults to true when the field is absent (older backend). */ mine: boolean + /** Server timestamp of the draft row, bumped on every draft update — a + * reliable per-row change marker for caches keyed on draft content. */ + created_at: string + /** Only set when listed with a `compareToWorkspace` (a fork comparing against + * its parent): true when this draft is identical to the parent's — cloned in + * on fork and never edited here. Undefined when no comparison was requested. */ + unchanged_from_parent?: boolean } export async function getDraftItems( workspace: string, - allUsers: boolean = false + allUsers: boolean = false, + compareToWorkspace?: string ): Promise { - const rows = await DraftService.listDrafts({ workspace, allUsers: allUsers || undefined }) + const rows = await DraftService.listDrafts({ + workspace, + allUsers: allUsers || undefined, + compareToWorkspace + }) return rows.map((r) => ({ kind: r.kind, path: r.path, @@ -64,7 +76,9 @@ export async function getDraftItems( raw_app: r.kind === 'raw_app', can_write: r.can_write ?? true, draft_users: r.draft_users, - mine: r.mine ?? true + mine: r.mine ?? true, + created_at: r.created_at, + unchanged_from_parent: r.unchanged_from_parent })) } @@ -77,6 +91,13 @@ export function invalidateWorkspaceDrafts(workspace: string | undefined): void { versions[workspace] = (versions[workspace] ?? 0) + 1 } +/** Current invalidation version for a workspace. Non-reactive read — callers + * compare it against a value captured earlier to detect Server-Draft mutations + * (any deploy/discard/draft write that called `invalidateWorkspaceDrafts`). */ +export function getWorkspaceDraftsVersion(workspace: string): number { + return versions[workspace] ?? 0 +} + export interface WorkspaceDraftsHandle { readonly items: DraftItem[] readonly count: number @@ -92,14 +113,15 @@ export interface WorkspaceDraftsHandle { */ export function useWorkspaceDrafts( workspace: () => string | undefined, - allUsers: () => boolean = () => false + allUsers: () => boolean = () => false, + compareToWorkspace: () => string | undefined = () => undefined ): WorkspaceDraftsHandle { const res = resource( () => { const ws = workspace() - return { ws, all: allUsers(), v: ws ? (versions[ws] ?? 0) : 0 } + return { ws, all: allUsers(), compare: compareToWorkspace(), v: ws ? (versions[ws] ?? 0) : 0 } }, - async ({ ws, all }) => (ws ? getDraftItems(ws, all) : []) + async ({ ws, all, compare }) => (ws ? getDraftItems(ws, all, compare) : []) ) return { get items() { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 8bd5119945..323f6b9ed7 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -28,6 +28,7 @@ workspaceUsageStore, userStore, workspaceStore, + userWorkspaces, type UserExt, defaultScripts, hubBaseUrlStore, @@ -37,7 +38,8 @@ devopsRole, whitelabelNameStore, globalDbManagerDrawer, - globalForkModal + globalForkModal, + globalS3FilePickerExplorer } from '$lib/stores' import CenteredModal from '$lib/components/CenteredModal.svelte' import { afterNavigate, beforeNavigate } from '$app/navigation' @@ -82,10 +84,12 @@ import WorkspaceScopeHeader from '$lib/components/sidebar/WorkspaceScopeHeader.svelte' import { DEFAULT_HUB_BASE_URL } from '$lib/hub' import DBManagerDrawer from '$lib/components/DBManagerDrawer.svelte' + import S3FilePicker from '$lib/components/S3FilePicker.svelte' import { useIsDarkMode } from '$lib/components/DarkModeObserver.svelte' import { useDbManagerUriState } from '$lib/components/dbManagerDrawerModel.svelte' import Modal2 from '$lib/components/common/modal/Modal2.svelte' import CreateWorkspaceInner from '$lib/components/workspaceSettings/CreateWorkspaceInner.svelte' + import { recordForkParent, rememberForkParent } from '$lib/forkParentMemory' interface Props { children?: import('svelte').Snippet } @@ -226,10 +230,10 @@ // nest the whole experience. Hide it when embedded. const embedded = BROWSER && window.self !== window.top - // AI sessions are still dev-gated (localStorage wm_dev_global_ai=1), same as - // the global chat. The Workspace ⇄ Sessions switch is the only entry point, so - // gate it on the flag too — otherwise it would ship the unfinished experience - // to prod. The /sessions page has its own gate for direct navigation. + // AI sessions (beta) are on unless the user opted out from the banner under + // the session chat. The Workspace ⇄ Sessions switch is the only entry point, + // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. + // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() if (page.status == 404) { @@ -518,6 +522,7 @@ nats_used, sqs_used, mqtt_used, + amqp_used, gcp_used, azure_used, email_used, @@ -545,6 +550,9 @@ if (mqtt_used) { usedKinds.push('mqtt') } + if (amqp_used) { + usedKinds.push('amqp') + } if (sqs_used) { usedKinds.push('sqs') } @@ -672,6 +680,40 @@ $workspaceStore untrack(() => updateUserStore($workspaceStore)) }) + // While a fork is reachable, mirror its parent linkage to localStorage so a + // later reload landing on a now-deleted fork can return to the parent (see + // forkParentMemory + the deleted-fork recovery in the root layout). + $effect(() => { + const ws = $workspaceStore + const list = $userWorkspaces + const isSuperadmin = $superadmin + untrack(() => void recordCurrentForkParent(ws, list, isSuperadmin)) + }) + + // A superadmin can open a fork they aren't a member of, including a prefixless + // dev workspace. The membership-gated list omits it, so `recordForkParent` can't + // see its parent — fetch it directly so the deleted-fork recovery still works. + async function recordCurrentForkParent( + ws: string | undefined, + list: typeof $userWorkspaces, + isSuperadmin: string | false | undefined + ): Promise { + if (!ws) return + if (list.some((w) => w.id === ws)) { + recordForkParent(ws, list) + return + } + if (!isSuperadmin) return + try { + const workspace = await WorkspaceService.getWorkspaceAsSuperAdmin({ workspace: ws }) + if (workspace.parent_workspace_id) { + rememberForkParent(ws, workspace.parent_workspace_id) + } + } catch { + // Best-effort: if we can't resolve the parent, recovery falls back to the + // workspace picker rather than the parent redirect. + } + } $effect(() => { $workspaceStore && untrack(() => onLoad()) }) @@ -738,6 +780,13 @@ }) globalDbManagerDrawer.val = useDbManagerUriState() + + let globalS3FilePicker: S3FilePicker | undefined = $state() + $effect(() => { + // `as any`: the component instance type is opaque in svelte2tsx context + // and does not match the store's structural type. + globalS3FilePickerExplorer.val = globalS3FilePicker as any + }) @@ -935,8 +984,8 @@ shortcut={`${getModifierKey()}k`} /> {#if !globalAiEnabled} - + aiChatManager.toggleOpen()} @@ -1068,8 +1117,8 @@ shortcut={`${getModifierKey()}k`} /> {#if !globalAiEnabled} - + aiChatManager.toggleOpen()} @@ -1262,10 +1311,14 @@ {/if} + {/if} +{#if $workspaceStore} + +{/if} + + import { getLocalDraftHint } from '$lib/localDraftHints.svelte' + import { run } from 'svelte/legacy' + + import { + AmqpTriggerService, + WorkspaceService, + type AmqpTrigger, + type TriggerMode, + type WorkspaceDeployUISettings + } from '$lib/gen' + import { + canWrite, + capitalize, + displayDate, + getLocalSetting, + sendUserToast, + storeLocalSetting, + removeTriggerKindIfUnused + } from '$lib/utils' + import { withForkConflictRetry } from '$lib/utils/forkConflict' + import { base } from '$app/paths' + import { page } from '$app/stores' + import CenteredPage from '$lib/components/CenteredPage.svelte' + import { Alert, Badge, Button, Skeleton } from '$lib/components/common' + import Dropdown from '$lib/components/DropdownV2.svelte' + import PageHeader from '$lib/components/PageHeader.svelte' + import SharedBadge from '$lib/components/SharedBadge.svelte' + import DraftBadge from '$lib/components/DraftBadge.svelte' + import ShareModal from '$lib/components/ShareModal.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { enterpriseLicense, usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' + import { Code, Eye, Pen, Plus, Shield, Trash, Circle, FileUp, Pause } from 'lucide-svelte' + import { goto } from '$lib/navigation' + import SearchItems from '$lib/components/SearchItems.svelte' + import NoItemFound from '$lib/components/home/NoItemFound.svelte' + import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import ListFilters from '$lib/components/home/ListFilters.svelte' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import { setQuery } from '$lib/navigation' + import { onDestroy, onMount } from 'svelte' + import Popover from '$lib/components/Popover.svelte' + import { isCloudHosted } from '$lib/cloud' + import AmqpTriggerEditor from '$lib/components/triggers/amqp/AmqpTriggerEditor.svelte' + import { AmqpIcon } from '$lib/components/icons' + import { ALL_DEPLOYABLE, isDeployable } from '$lib/utils_deployable' + import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' + import TriggerModeToggle from '$lib/components/triggers/TriggerModeToggle.svelte' + + type TriggerM = AmqpTrigger & { canWrite: boolean } + + let triggers: TriggerM[] = $state([]) + let shareModal: ShareModal | undefined = $state() + let loading = $state(true) + let deploymentDrawer: DeployWorkspaceDrawer | undefined = $state() + let deployUiSettings: WorkspaceDeployUISettings | undefined = $state(undefined) + + async function getDeployUiSettings() { + if (!$enterpriseLicense) { + deployUiSettings = ALL_DEPLOYABLE + return + } + let settings = await WorkspaceService.getPublicSettings({ workspace: $workspaceStore! }) + deployUiSettings = settings.deploy_ui ?? ALL_DEPLOYABLE + } + getDeployUiSettings() + + async function loadTriggers(): Promise { + triggers = (await AmqpTriggerService.listAmqpTriggers({ workspace: $workspaceStore!, includeDraftOnly: true })).map( + (x) => { + return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } + } + ) + $usedTriggerKinds = removeTriggerKindIfUnused(triggers.length, 'amqp', $usedTriggerKinds) + loading = false + } + + let interval = setInterval(async () => { + try { + const newTriggers = await AmqpTriggerService.listAmqpTriggers({ + workspace: $workspaceStore! + }) + for (let i = 0; i < triggers.length; i++) { + const newTrigger = newTriggers.find((x) => x.path === triggers[i].path) + if (newTrigger) { + triggers[i] = { + ...triggers[i], + error: newTrigger.error, + last_server_ping: newTrigger.last_server_ping, + mode: newTrigger.mode, + server_id: newTrigger.server_id + } + } + } + } catch (err) { + console.error(err) + } + }, 5000) + + onDestroy(() => { + clearInterval(interval) + }) + + async function onToggleMode(path: string, mode: TriggerMode): Promise { + let committed = false + try { + const ok = await withForkConflictRetry( + (force) => + AmqpTriggerService.setAmqpTriggerMode({ + path, + workspace: $workspaceStore!, + requestBody: { mode, force } + }), + 'AMQP trigger' + ) + if (ok) { + sendUserToast(`${capitalize(mode)} AMQP trigger ${path}`) + loadTriggers() + } + committed = ok + } catch (err) { + sendUserToast( + `Cannot ${mode === 'enabled' ? 'enable' : mode === 'disabled' ? 'disable' : 'suspend'} amqp trigger: ${err.body}`, + true + ) + loadTriggers() + } + return committed + } + + run(() => { + if ($workspaceStore && $userStore) { + loadTriggers() + } + }) + let amqpTriggerEditor: AmqpTriggerEditor | undefined = $state() + + let hashHandled = false + $effect(() => { + if (!hashHandled && triggers.length > 0 && amqpTriggerEditor) { + let hash = $page.url.hash + if (hash.length > 1) { + let path = hash.slice(1) + let trigger = triggers.find((t) => t.path === path) + if (trigger) { + hashHandled = true + amqpTriggerEditor?.openEdit(path, trigger.is_flow) + } + } + } + }) + + let filteredItems: (TriggerM & { marked?: any })[] | undefined = $state([]) + let items: typeof filteredItems | undefined = $state([]) + let preFilteredItems: typeof filteredItems | undefined = $state([]) + let filter = $state('') + let ownerFilter: string | undefined = $state(undefined) + let nbDisplayed = $state(15) + + const TRIGGER_PATH_KIND_FILTER_SETTING = 'filter_path_of' + const FILTER_USER_FOLDER_SETTING_NAME = 'user_and_folders_only' + let selectedFilterKind = $state( + (getLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING) as 'trigger' | 'script_flow') ?? 'trigger' + ) + let filterUserFolders = $state(getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true') + + run(() => { + storeLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING, selectedFilterKind) + }) + run(() => { + storeLocalSetting(FILTER_USER_FOLDER_SETTING_NAME, filterUserFolders ? 'true' : undefined) + }) + + function filterItemsPathsBaseOnUserFilters( + item: TriggerM, + selectedFilterKind: 'trigger' | 'script_flow', + filterUserFolders: boolean + ) { + if ($workspaceStore == 'admins') return true + if (filterUserFolders) { + if (selectedFilterKind === 'trigger') { + return ( + !item.path.startsWith('u/') || item.path.startsWith('u/' + $userStore?.username + '/') + ) + } else { + return ( + !item.script_path.startsWith('u/') || + item.script_path.startsWith('u/' + $userStore?.username + '/') + ) + } + } else { + return true + } + } + + run(() => { + preFilteredItems = + ownerFilter != undefined + ? selectedFilterKind === 'trigger' + ? triggers?.filter( + (x) => + x.path.startsWith(ownerFilter + '/') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter( + (x) => + x.script_path.startsWith(ownerFilter + '/') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter((x) => + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + }) + + run(() => { + if ($workspaceStore) { + ownerFilter = undefined + } + }) + + let owners = $derived( + selectedFilterKind === 'trigger' + ? Array.from( + new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + : Array.from( + new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + ) + + run(() => { + items = filter !== '' ? filteredItems : preFilteredItems + }) + + function updateQueryFilters(selectedFilterKind, filterUserFolders) { + setQuery( + new URL(window.location.href), + TRIGGER_PATH_KIND_FILTER_SETTING, + selectedFilterKind, + window.location.hash || undefined + ).then(() => { + setQuery( + new URL(window.location.href), + FILTER_USER_FOLDER_SETTING_NAME, + String(filterUserFolders), + window.location.hash || undefined + ) + }) + } + + function loadQueryFilters() { + let url = new URL(window.location.href) + let queryFilterKind = url.searchParams.get(TRIGGER_PATH_KIND_FILTER_SETTING) + let queryFilterUserFolders = url.searchParams.get(FILTER_USER_FOLDER_SETTING_NAME) + if (queryFilterKind) { + selectedFilterKind = queryFilterKind as 'trigger' | 'script_flow' + } + if (queryFilterUserFolders) { + filterUserFolders = queryFilterUserFolders == 'true' + } + } + + onMount(() => { + loadQueryFilters() + }) + + run(() => { + updateQueryFilters(selectedFilterKind, filterUserFolders) + }) + + + + + + (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} +/> + + + + + + + {#if isCloudHosted()} + + AMQP triggers are disabled in the multi-tenant cloud. + +
+ {/if} +
+
+ +
+
Filter by path of
+ + {#snippet children({ item })} + + + {/snippet} + +
+ + +
+ {#if $userStore?.is_super_admin && $userStore.username.includes('@')} + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
+
+ {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} +
No AMQP triggers
+ {:else if items?.length} +
+ {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, is_flow, extra_perms, canWrite, error, last_server_ping, server_id, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} + {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} + {@const enabled = mode === 'enabled' || mode === 'suspended'} + +
+
+ + + amqpTriggerEditor?.openEdit(path, is_flow)} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
+ {path}{(getLocalDraftHint($workspaceStore, 'trigger_amqp', path) ?? is_draft) ? '*' : ''} +
+
+ runnable: {script_path} +
+
+ + + +
+ {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} + + + + + + {#snippet text()} +
+ {#if enabled} + {#if !server_id} + AMQP trigger is starting... + {:else} + AMQP trigger is not connected{error ? ': ' + error : ''} + {/if} + {:else} + AMQP trigger was disabled because of an error: {error} + {/if} +
+ {/snippet} +
+ {:else if enabled} + + + + + {#snippet text()} +
+ AMQP trigger is connected{!server_id ? ' (shutting down...)' : ''} +
+ {/snippet} +
+ {/if} +
+ + onToggleMode(path, newMode)} + triggerMode={mode} + includeModalConfig={{ + triggerPath: path, + triggerKind: 'amqp', + runnableConfig: { + path: script_path, + kind: is_flow ? 'flow' : 'script', + retry, + errorHandlerPath: error_handler_path, + errorHandlerArgs: error_handler_args + } + }} + {canWrite} + hideToggleLabels + hideDropdown + /> + +
+ + { + goto(href) + } + }, + ...(canWrite && mode !== 'suspended' + ? [ + { + displayName: 'Suspend job execution', + icon: Pause, + action: () => { + onToggleMode(path, 'suspended') + } + } + ] + : []), + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + amqpTriggerEditor?.openEdit(path, is_flow) + } + }, + ...(isDeployable('trigger', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'trigger', { + triggers: { + kind: 'amqp' + } + }) + } + } + ] + : []), + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: 'Permissions', + icon: Shield, + action: () => { + shareModal?.openDrawer(path, 'amqp_trigger') + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite, + action: async () => { + await AmqpTriggerService.deleteAmqpTrigger({ + workspace: $workspaceStore ?? '', + path + }) + loadTriggers() + } + } + ]} + /> +
+
+
+
edited by {edited_by}
the {displayDate(edited_at)}
+
+ {/each} +
+ {:else} + + {/if} +
+ {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} +
+ + { + loadTriggers() + }} +/> diff --git a/frontend/src/routes/(root)/(logged)/dev/session-tree/+page.svelte b/frontend/src/routes/(root)/(logged)/dev/session-tree/+page.svelte index eb69dc1969..1bd8b0879b 100644 --- a/frontend/src/routes/(root)/(logged)/dev/session-tree/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/dev/session-tree/+page.svelte @@ -2,8 +2,7 @@ // DEV-ONLY design exploration: how to visually distinguish a *workspace* from a // *session* in the rail tree, and make the hierarchy (forks) clear. Not linked // anywhere; open at /dev/session-tree. Pure mock data, no real session state. - // Gated behind the same dev flag as the sessions UI it explores. - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + // Only reachable on dev builds. import WorkspaceIcon from '$lib/components/workspace/WorkspaceIcon.svelte' import { MessageSquare, @@ -323,7 +322,7 @@ {/snippet} -{#if isGlobalAiEnabled()} +{#if import.meta.env.DEV}

diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index 0a55acfe69..0df2235212 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -14,17 +14,25 @@ import { sendUserToast } from '$lib/utils' import DataTable from '$lib/components/table/DataTable.svelte' import Cell from '$lib/components/table/Cell.svelte' - import { Pen, Trash, Plus } from 'lucide-svelte' + import { Pen, Trash, Plus, UploadCloud } from 'lucide-svelte' + import DeployToHub from '$lib/components/workspaceSettings/DeployToHub.svelte' import Head from '$lib/components/table/Head.svelte' import Row from '$lib/components/table/Row.svelte' import Badge from '$lib/components/common/badge/Badge.svelte' import { untrack } from 'svelte' + import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' type FolderW = Folder & { canWrite: boolean } + let restricted = $derived( + isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + ) + let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) let folderDrawer: Drawer | undefined = $state() + let hubDrawer: Drawer | undefined = $state() + let publishFolderName: string = $state('') async function loadFolders(): Promise { folders = (await FolderService.listFolders({ workspace: $workspaceStore! })).map((x) => { @@ -83,6 +91,22 @@ + + { + hubDrawer?.closeDrawer() + publishFolderName = '' + }} + > + {#if publishFolderName} + {#key publishFolderName} + + {/key} + {/if} + + + {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.folders}